Pulkitg64 commented on code in PR #16739:
URL: https://github.com/apache/lucene/pull/16739#discussion_r4140937891
##########
lucene/core/src/java/org/apache/lucene/util/hnsw/IncrementalHnswGraphMerger.java:
##########
@@ -64,9 +64,14 @@ public class IncrementalHnswGraphMerger implements
HnswGraphMerger {
*/
private final int DELETE_PCT_THRESHOLD = 40;
- /** Represents a vector reader that contains graph info. */
+ /**
+ * Represents a vector reader that contains graph info.
+ *
Review Comment:
While you are here, let's add params for the other members as well here for
completeness?
##########
lucene/CHANGES.txt:
##########
@@ -179,6 +179,11 @@ Improvements
Optimizations
---------------------
+* GITHUB#16739: IncrementalHnswGraphMerger now picks the base graph for a
merge by comparing live
Review Comment:
I think this should go into Bug Fixes (But I am not sure so no strong
opinion).
##########
lucene/core/src/test/org/apache/lucene/util/hnsw/TestIncrementalHnswGraphMergerBaseSelection.java:
##########
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util.hnsw;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.KnnFloatVectorField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.CodecReader;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+
+/**
+ * Tests which source graph {@link IncrementalHnswGraphMerger} picks as the
base graph when some
+ * candidates carry deletions.
+ */
+public class TestIncrementalHnswGraphMergerBaseSelection extends
LuceneTestCase {
+
+ private static final int DIM = 16;
+ private static final String FIELD = "v";
+
+ /**
+ * Segment A has 1000 nodes of which 300 are deleted (700 live, 30% deleted,
under the 40%
+ * threshold); segment B has 900 nodes and no deletions. B holds more live
vectors, so it must be
+ * chosen as the base regardless of the order in which the readers are added.
+ */
+ public void testBaseGraphChosenByLiveCount() throws IOException {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig cfg = new IndexWriterConfig();
+ cfg.setCodec(TestUtil.alwaysKnnVectorsFormat(new
Lucene99HnswVectorsFormat(16, 100, 0)));
+ cfg.setMergePolicy(NoMergePolicy.INSTANCE);
+ try (IndexWriter w = new IndexWriter(dir, cfg)) {
+ addSegment(w, "a", 1000);
+ addSegment(w, "b", 900);
+ for (int i = 0; i < 300; i++) {
+ w.deleteDocuments(new Term("id", "a" + i));
+ }
+ w.commit();
+ }
+
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ List<LeafReaderContext> leaves = reader.leaves();
+ assertEquals(2, leaves.size());
+ CodecReader a = segment(leaves, 1000);
+ CodecReader b = segment(leaves, 900);
+ assertEquals(700, a.numDocs());
+ assertEquals(900, b.numDocs());
+ FieldInfo fieldInfo = a.getFieldInfos().fieldInfo(FIELD);
+
+ for (boolean aFirst : new boolean[] {true, false}) {
+ for (boolean concurrent : new boolean[] {false, true}) {
+ IncrementalHnswGraphMerger merger =
+ concurrent
+ ? new ConcurrentHnswMerger(fieldInfo, null, 16, 100, null,
2)
+ : new IncrementalHnswGraphMerger(fieldInfo, null, 16, 100);
Review Comment:
Should we move the constant from here as well i.e M and beamWidth?
##########
lucene/core/src/test/org/apache/lucene/util/hnsw/TestIncrementalHnswGraphMergerBaseSelection.java:
##########
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util.hnsw;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.KnnFloatVectorField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.CodecReader;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+
+/**
+ * Tests which source graph {@link IncrementalHnswGraphMerger} picks as the
base graph when some
+ * candidates carry deletions.
+ */
+public class TestIncrementalHnswGraphMergerBaseSelection extends
LuceneTestCase {
+
+ private static final int DIM = 16;
+ private static final String FIELD = "v";
+
+ /**
+ * Segment A has 1000 nodes of which 300 are deleted (700 live, 30% deleted,
under the 40%
+ * threshold); segment B has 900 nodes and no deletions. B holds more live
vectors, so it must be
+ * chosen as the base regardless of the order in which the readers are added.
+ */
+ public void testBaseGraphChosenByLiveCount() throws IOException {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig cfg = new IndexWriterConfig();
+ cfg.setCodec(TestUtil.alwaysKnnVectorsFormat(new
Lucene99HnswVectorsFormat(16, 100, 0)));
+ cfg.setMergePolicy(NoMergePolicy.INSTANCE);
+ try (IndexWriter w = new IndexWriter(dir, cfg)) {
+ addSegment(w, "a", 1000);
Review Comment:
Should we make constant variable as we are using same value across multiple
places?
```
private static final int A_DOCS = 1000;
private static final int B_DOCS = 900;
private static final String FIELD_NAME = "id";
```
##########
lucene/core/src/test/org/apache/lucene/util/hnsw/TestIncrementalHnswGraphMergerBaseSelection.java:
##########
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util.hnsw;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.KnnFloatVectorField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.CodecReader;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+
+/**
+ * Tests which source graph {@link IncrementalHnswGraphMerger} picks as the
base graph when some
+ * candidates carry deletions.
+ */
+public class TestIncrementalHnswGraphMergerBaseSelection extends
LuceneTestCase {
Review Comment:
Non blocking: Should we make the test more robust which covers more scenario
and not just `live(A) < live(B) with total(A) > total(B)`?
##########
lucene/core/src/test/org/apache/lucene/util/hnsw/TestIncrementalHnswGraphMergerBaseSelection.java:
##########
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util.hnsw;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.KnnFloatVectorField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.CodecReader;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+
+/**
+ * Tests which source graph {@link IncrementalHnswGraphMerger} picks as the
base graph when some
+ * candidates carry deletions.
+ */
+public class TestIncrementalHnswGraphMergerBaseSelection extends
LuceneTestCase {
+
+ private static final int DIM = 16;
+ private static final String FIELD = "v";
+
+ /**
+ * Segment A has 1000 nodes of which 300 are deleted (700 live, 30% deleted,
under the 40%
+ * threshold); segment B has 900 nodes and no deletions. B holds more live
vectors, so it must be
+ * chosen as the base regardless of the order in which the readers are added.
+ */
+ public void testBaseGraphChosenByLiveCount() throws IOException {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig cfg = new IndexWriterConfig();
+ cfg.setCodec(TestUtil.alwaysKnnVectorsFormat(new
Lucene99HnswVectorsFormat(16, 100, 0)));
+ cfg.setMergePolicy(NoMergePolicy.INSTANCE);
+ try (IndexWriter w = new IndexWriter(dir, cfg)) {
+ addSegment(w, "a", 1000);
+ addSegment(w, "b", 900);
+ for (int i = 0; i < 300; i++) {
Review Comment:
I think we are trying to keep the delete% under 40% which is set in
```IncrementalHnswGraphMerger.java.DELETE_PCT_THRESHOLD```. What is that
threshold changes, this can make the test case buggy as it can fail/succeed for
wrong reasons.
We should derive the delete% here based on
`IncrementalHnswGraphMerger.java.DELETE_PCT_THRESHOLD` instead of hardcoding.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
