From 69fc5cea53d2d72f3593318f3ece94368378228a Mon Sep 17 00:00:00 2001 From: Brett Wooldridge Date: Fri, 4 Sep 2026 12:40:25 +0900 Subject: [PATCH] perf: stream equality and range index scans instead of materializing every id NitriteIndexer.findByFilter returns a LinkedHashSet of every matching id, so find(k = v).firstOrNull() built the whole match set before handing back one row, and a bounded page paid for the entire result. On a non-unique index over a low-cardinality field that set is a large fraction of the collection on every lookup. The composite layout already keeps its rows in key order, so the two plan shapes that map onto one bounded walk of it, an equality on the indexed field and a two-sided range on it, are now served by a lazy iterator that starts at the first key inside the bounds and stops at the first key outside them. It honours the plan's reverse scan order by visiting the key groups backwards while reading each group forwards, exactly as the materialized scan orders them, skips entries removed in an open transaction, and returns a document indexed under several keys once. NitriteIndex.findNitriteIdStream and NitriteIndexer.findByFilterStream are new default methods returning null, so every other index type, plugin indexer and plan shape keeps the materialized path unchanged. ReadOperations prefers the stream when one is offered; the covered-count shortcut that lets size() answer without fetching documents is kept by counting the streamed ids on demand, so size() still reads the index only. Tests compare the stream with the materialized scan for equality, range and reverse order, check the shapes it declines, show with a spied map that only one key is read for the first row, and exercise counts, paging, descending order, multi-valued fields and removals through the public API. Co-Authored-By: Claude Fable 5.1 --- .../collection/operation/ReadOperations.java | 67 ++++--- .../no2/common/streams/DocumentStream.java | 12 ++ .../no2/common/streams/IndexedStream.java | 19 +- .../dizitart/no2/index/ComparableIndexer.java | 7 + .../org/dizitart/no2/index/NitriteIndex.java | 14 ++ .../dizitart/no2/index/NitriteIndexer.java | 14 ++ .../dizitart/no2/index/SingleFieldIndex.java | 185 ++++++++++++++++++ .../no2/collection/LazyIndexScanTest.java | 103 ++++++++++ .../no2/index/SingleFieldIndexTest.java | 96 ++++++++- 9 files changed, 487 insertions(+), 30 deletions(-) create mode 100644 nitrite/src/test/java/org/dizitart/no2/collection/LazyIndexScanTest.java diff --git a/nitrite/src/main/java/org/dizitart/no2/collection/operation/ReadOperations.java b/nitrite/src/main/java/org/dizitart/no2/collection/operation/ReadOperations.java index 7d9c838e..f3f7304d 100644 --- a/nitrite/src/main/java/org/dizitart/no2/collection/operation/ReadOperations.java +++ b/nitrite/src/main/java/org/dizitart/no2/collection/operation/ReadOperations.java @@ -111,35 +111,43 @@ private void prepareLogicalFilter(LogicalFilter logicalFilter) { } private DocumentCursor createCursor(FindPlan findPlan) { - // -1 means "not an index scan"; the index branch records the exact id-set size here. - long[] indexedIdCount = { -1 }; - RecordStream> recordStream = findSuitableStream(findPlan, indexedIdCount); + IndexScan scan = new IndexScan(); + RecordStream> recordStream = findSuitableStream(findPlan, scan); DocumentStream cursor = new DocumentStream(recordStream, processorChain); cursor.setFindPlan(findPlan); - cursor.setCoveredCount(computeCoveredCount(findPlan, indexedIdCount[0])); + if (isCountCovered(findPlan)) { + if (findPlan.getIndexDescriptor() == null) { + // pure full scan over the whole collection + cursor.setCoveredCount(nitriteMap.size()); + } else if (scan.lazyStream != null) { + // the ids are read lazily: count them from the index only if size() is asked + cursor.setCoveredCountSupplier(scan.lazyStream::countIds); + } else if (scan.idCount >= 0) { + // the index supplied the exact matching id set + cursor.setCoveredCount(scan.idCount); + } + } return cursor; } + /** What an index scan handed back: an exact id count, or the lazy stream, or neither. */ + private static final class IndexScan { + private long idCount = -1; + private IndexedStream lazyStream; + } + /** * Returns the exact match count when the query is fully answered without fetching documents, * or {@code null} when the cursor must be drained to count. The count is exact only when * nothing downstream drops or changes cardinality (a post-filter, skip, or limit); sort does * not change the count, and an OR-union needs de-duplication so its count cannot be derived. */ - private Long computeCoveredCount(FindPlan findPlan, long indexedIdCount) { - if (!findPlan.getSubPlans().isEmpty() - || findPlan.getCollectionScanFilter() != null - || findPlan.getSkip() != null - || findPlan.getLimit() != null - || findPlan.getByIdFilter() != null) { - return null; - } - if (findPlan.getIndexDescriptor() != null) { - // the index supplied the exact matching id set - return indexedIdCount >= 0 ? indexedIdCount : null; - } - // pure full scan over the whole collection - return nitriteMap.size(); + private boolean isCountCovered(FindPlan findPlan) { + return findPlan.getSubPlans().isEmpty() + && findPlan.getCollectionScanFilter() == null + && findPlan.getSkip() == null + && findPlan.getLimit() == null + && findPlan.getByIdFilter() == null; } /** @@ -193,7 +201,7 @@ private static Object indexedValue(DBValue dbValue) { return dbValue == null || dbValue instanceof DBNull ? null : dbValue.getValue(); } - private RecordStream> findSuitableStream(FindPlan findPlan, long[] indexedIdCount) { + private RecordStream> findSuitableStream(FindPlan findPlan, IndexScan scan) { RecordStream> rawStream; RecordStream> indexSortedStream = null; @@ -202,7 +210,7 @@ private RecordStream> findSuitableStream(FindPlan find List>> subStreams = new ArrayList<>(); for (FindPlan subPlan : findPlan.getSubPlans()) { // a sub-plan's own id count cannot answer the union's count (dedup), so discard it - RecordStream> suitableStream = findSuitableStream(subPlan, new long[]{ -1 }); + RecordStream> suitableStream = findSuitableStream(subPlan, new IndexScan()); subStreams.add(suitableStream); } @@ -233,14 +241,21 @@ private RecordStream> findSuitableStream(FindPlan find if (indexDescriptor != null) { // get optimized filter NitriteIndexer indexer = nitriteConfig.findIndexer(indexDescriptor.getIndexType()); - LinkedHashSet nitriteIds = indexer.findByFilter(findPlan, nitriteConfig); + RecordStream idStream = indexer.findByFilterStream(findPlan, nitriteConfig); + if (idStream != null) { + // the index walks its matches lazily; a size() is counted from it on demand + scan.lazyStream = new IndexedStream(idStream, nitriteMap); + rawStream = scan.lazyStream; + } else { + LinkedHashSet nitriteIds = indexer.findByFilter(findPlan, nitriteConfig); - // the index supplied the exact matching id set; record its size so a size() - // with no row-dropping step downstream can answer from it without fetching - indexedIdCount[0] = nitriteIds.size(); + // the index supplied the exact matching id set; record its size so a size() + // with no row-dropping step downstream can answer from it without fetching + scan.idCount = nitriteIds.size(); - // create indexed stream from optimized filter - rawStream = new IndexedStream(nitriteIds, nitriteMap); + // create indexed stream from optimized filter + rawStream = new IndexedStream(nitriteIds, nitriteMap); + } } else { indexSortedStream = indexSortedStream(findPlan); rawStream = indexSortedStream != null ? indexSortedStream : nitriteMap.entries(); diff --git a/nitrite/src/main/java/org/dizitart/no2/common/streams/DocumentStream.java b/nitrite/src/main/java/org/dizitart/no2/common/streams/DocumentStream.java index e2b2e175..6dd03171 100644 --- a/nitrite/src/main/java/org/dizitart/no2/common/streams/DocumentStream.java +++ b/nitrite/src/main/java/org/dizitart/no2/common/streams/DocumentStream.java @@ -33,6 +33,7 @@ import java.util.Collections; import java.util.Iterator; +import java.util.function.LongSupplier; /** * @since 4.0 @@ -53,6 +54,13 @@ public class DocumentStream implements DocumentCursor { @Setter private Long coveredCount; + /** + * Answers {@link #size()} from the index on demand when the match count is known to be + * covered but the ids are streamed lazily rather than materialized; evaluated once. + */ + @Setter + private LongSupplier coveredCountSupplier; + public DocumentStream(RecordStream> recordStream, ProcessorChain processorChain) { this.recordStream = recordStream; @@ -64,6 +72,10 @@ public long size() { if (coveredCount != null) { return coveredCount; } + if (coveredCountSupplier != null) { + coveredCount = coveredCountSupplier.getAsLong(); + return coveredCount; + } return Iterables.size(this); } diff --git a/nitrite/src/main/java/org/dizitart/no2/common/streams/IndexedStream.java b/nitrite/src/main/java/org/dizitart/no2/common/streams/IndexedStream.java index be6c2335..e79d8fdb 100644 --- a/nitrite/src/main/java/org/dizitart/no2/common/streams/IndexedStream.java +++ b/nitrite/src/main/java/org/dizitart/no2/common/streams/IndexedStream.java @@ -24,7 +24,6 @@ import org.dizitart.no2.store.NitriteMap; import java.util.Iterator; -import java.util.Set; /** * @author Anindya Chatterjee @@ -32,9 +31,9 @@ */ public class IndexedStream implements RecordStream> { private final NitriteMap nitriteMap; - private final Set nitriteIds; + private final Iterable nitriteIds; - public IndexedStream(Set nitriteIds, + public IndexedStream(Iterable nitriteIds, NitriteMap nitriteMap) { this.nitriteIds = nitriteIds; this.nitriteMap = nitriteMap; @@ -45,6 +44,20 @@ public Iterator> iterator() { return new IndexedStreamIterator(nitriteIds.iterator(), nitriteMap); } + /** + * Counts the ids the index supplied, walking the id source only, without fetching a + * single document. + * + * @return the number of ids + */ + public long countIds() { + long count = 0; + for (NitriteId ignored : nitriteIds) { + count++; + } + return count; + } + private static class IndexedStreamIterator implements Iterator>, SkippableIterator { private final Iterator iterator; diff --git a/nitrite/src/main/java/org/dizitart/no2/index/ComparableIndexer.java b/nitrite/src/main/java/org/dizitart/no2/index/ComparableIndexer.java index 49630d1c..45a03d3c 100644 --- a/nitrite/src/main/java/org/dizitart/no2/index/ComparableIndexer.java +++ b/nitrite/src/main/java/org/dizitart/no2/index/ComparableIndexer.java @@ -21,6 +21,7 @@ import org.dizitart.no2.collection.NitriteId; import org.dizitart.no2.common.DBValue; import org.dizitart.no2.common.FieldValues; +import org.dizitart.no2.common.RecordStream; import org.dizitart.no2.common.Fields; import org.dizitart.no2.common.tuples.Pair; import org.dizitart.no2.exceptions.IndexingException; @@ -66,6 +67,12 @@ public LinkedHashSet findByFilter(FindPlan findPlan, NitriteConfig ni return nitriteIndex.findNitriteIds(findPlan); } + @Override + public RecordStream findByFilterStream(FindPlan findPlan, NitriteConfig nitriteConfig) { + NitriteIndex nitriteIndex = findNitriteIndex(findPlan.getIndexDescriptor(), nitriteConfig); + return nitriteIndex.findNitriteIdStream(findPlan); + } + @Override public List> readSortKeys(IndexDescriptor indexDescriptor, NitriteConfig nitriteConfig, diff --git a/nitrite/src/main/java/org/dizitart/no2/index/NitriteIndex.java b/nitrite/src/main/java/org/dizitart/no2/index/NitriteIndex.java index 45dedf97..b8a38a19 100644 --- a/nitrite/src/main/java/org/dizitart/no2/index/NitriteIndex.java +++ b/nitrite/src/main/java/org/dizitart/no2/index/NitriteIndex.java @@ -21,6 +21,7 @@ import org.dizitart.no2.collection.NitriteId; import org.dizitart.no2.common.DBValue; import org.dizitart.no2.common.FieldValues; +import org.dizitart.no2.common.RecordStream; import org.dizitart.no2.common.tuples.Pair; import org.dizitart.no2.exceptions.UniqueConstraintException; import org.dizitart.no2.exceptions.ValidationException; @@ -74,6 +75,19 @@ public interface NitriteIndex { */ LinkedHashSet findNitriteIds(FindPlan findPlan); + /** + * Streams the ids matching the plan lazily, in index order and without duplicates, or + * returns {@code null} when this index cannot do so for the given plan, in which case the + * caller falls back to {@link #findNitriteIds(FindPlan)}. A stream lets a query that only + * needs the first rows, or a bounded page, stop reading the index as soon as it has them. + * + * @param findPlan the find plan + * @return a re-iterable stream of ids, or {@code null} + */ + default RecordStream findNitriteIdStream(FindPlan findPlan) { + return null; + } + /** * Reads every {@code (indexed value, id)} pair out of the index, so a sorted query can * decide its order without deserializing a single document. diff --git a/nitrite/src/main/java/org/dizitart/no2/index/NitriteIndexer.java b/nitrite/src/main/java/org/dizitart/no2/index/NitriteIndexer.java index ba8085f6..bc51fa35 100644 --- a/nitrite/src/main/java/org/dizitart/no2/index/NitriteIndexer.java +++ b/nitrite/src/main/java/org/dizitart/no2/index/NitriteIndexer.java @@ -21,6 +21,7 @@ import org.dizitart.no2.collection.NitriteId; import org.dizitart.no2.common.DBValue; import org.dizitart.no2.common.FieldValues; +import org.dizitart.no2.common.RecordStream; import org.dizitart.no2.common.Fields; import org.dizitart.no2.common.module.NitritePlugin; import org.dizitart.no2.common.tuples.Pair; @@ -88,6 +89,19 @@ public interface NitriteIndexer extends NitritePlugin { */ LinkedHashSet findByFilter(FindPlan findPlan, NitriteConfig nitriteConfig); + /** + * Streams the ids matching the plan lazily, or returns {@code null} when the indexer has no + * lazy path for it and {@link #findByFilter(FindPlan, NitriteConfig)} must be used. The + * default is {@code null}, so existing indexer plugins are unaffected. + * + * @param findPlan the find plan + * @param nitriteConfig the nitrite config + * @return a re-iterable stream of ids, or {@code null} + */ + default RecordStream findByFilterStream(FindPlan findPlan, NitriteConfig nitriteConfig) { + return null; + } + /** * Reads every {@code (indexed value, id)} pair out of the given index, so a sorted query * can decide its order without deserializing a single document. diff --git a/nitrite/src/main/java/org/dizitart/no2/index/SingleFieldIndex.java b/nitrite/src/main/java/org/dizitart/no2/index/SingleFieldIndex.java index 8b86af74..73eba854 100644 --- a/nitrite/src/main/java/org/dizitart/no2/index/SingleFieldIndex.java +++ b/nitrite/src/main/java/org/dizitart/no2/index/SingleFieldIndex.java @@ -26,6 +26,9 @@ import org.dizitart.no2.common.Fields; import org.dizitart.no2.common.tuples.Pair; import org.dizitart.no2.filters.ComparableFilter; +import org.dizitart.no2.common.RecordStream; +import org.dizitart.no2.filters.EqualsFilter; +import org.dizitart.no2.filters.SortingAwareFilter; import org.dizitart.no2.store.NitriteMap; import org.dizitart.no2.store.NitriteStore; @@ -33,6 +36,8 @@ import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; +import java.util.NoSuchElementException; +import java.util.Iterator; import java.util.Set; import static org.dizitart.no2.common.util.IndexUtils.deriveCompositeIndexMapName; @@ -133,6 +138,186 @@ public LinkedHashSet findNitriteIds(FindPlan findPlan) { return scanIndex(findPlan, iMap); } + /** + * A lazy id stream for the two plan shapes the composite layout answers with one bounded + * walk of the map: an equality on the indexed field, and a two-sided range on it. The walk + * starts at the first key inside the bounds and stops at the first key outside them, so a + * caller that wants the first row, or a page, reads only that far. Every other shape, and + * the unique layout, returns {@code null} and is served by {@link #findNitriteIds(FindPlan)}. + */ + @Override + public RecordStream findNitriteIdStream(FindPlan findPlan) { + if (!useCompositeLayout() || findPlan.getIndexScanFilter() == null) { + return null; + } + List filters = findPlan.getIndexScanFilter().getFilters(); + Range range = Range.of(filters); + if (range == null) { + return null; + } + String field = filters.get(0).getField(); + boolean reverse = findPlan.getIndexScanOrder() != null + && Boolean.TRUE.equals(findPlan.getIndexScanOrder().get(field)); + NitriteMap compositeMap = findCompositeMap(); + return RecordStream.fromIterable(() -> new CompositeRangeIterator(compositeMap, range, reverse)); + } + + /** Inclusive-or-exclusive bounds on the indexed value; {@code null} when a plan has another shape. */ + private static final class Range { + private final DBValue lower; + private final boolean lowerInclusive; + private final DBValue upper; + private final boolean upperInclusive; + + private Range(DBValue lower, boolean lowerInclusive, DBValue upper, boolean upperInclusive) { + this.lower = lower; + this.lowerInclusive = lowerInclusive; + this.upper = upper; + this.upperInclusive = upperInclusive; + } + + static Range of(List filters) { + if (filters == null || filters.isEmpty()) { + return null; + } + if (filters.size() == 1 && filters.get(0).getClass() == EqualsFilter.class) { + Object value = filters.get(0).getValue(); + if (value == null) { + return new Range(DBNull.getInstance(), true, DBNull.getInstance(), true); + } + if (!(value instanceof Comparable)) { + return null; + } + DBValue key = new DBValue((Comparable) value); + return new Range(key, true, key, true); + } + + // a two-sided range on one field, the same shape IndexScanner.scanBoundedRange takes + String field = filters.get(0).getField(); + DBValue lower = null, upper = null; + boolean lowerInclusive = false, upperInclusive = false; + for (ComparableFilter filter : filters) { + if (!(filter instanceof SortingAwareFilter) || field == null || !field.equals(filter.getField())) { + return null; + } + Object value = filter.getValue(); + if (!(value instanceof Comparable)) { + return null; + } + DBValue key = new DBValue((Comparable) value); + switch (((SortingAwareFilter) filter).getComparisonMode()) { + case GreaterEqual: + if (lower != null) return null; + lower = key; lowerInclusive = true; break; + case Greater: + if (lower != null) return null; + lower = key; lowerInclusive = false; break; + case LesserEqual: + if (upper != null) return null; + upper = key; upperInclusive = true; break; + case Lesser: + if (upper != null) return null; + upper = key; upperInclusive = false; break; + default: + return null; + } + } + return lower == null || upper == null ? null : new Range(lower, lowerInclusive, upper, upperInclusive); + } + } + + /** + * Walks the composite map between the bounds, in index order or in reverse, skipping + * entries removed in an open transaction and ids already returned (a multi-valued field + * indexes one document under several keys). Ids sharing a key are always returned in their + * stored order, so a reverse walk visits the key groups backwards but reads each group + * forwards, exactly as the materialized scan orders them. + */ + private static final class CompositeRangeIterator implements Iterator { + private final NitriteMap map; + private final Range range; + private final boolean reverse; + private final Set seen = new HashSet<>(); + private final java.util.ArrayDeque group = new java.util.ArrayDeque<>(); + private IndexEntryKey key; + private NitriteId next; + private boolean started; + + CompositeRangeIterator(NitriteMap map, Range range, boolean reverse) { + this.map = map; + this.range = range; + this.reverse = reverse; + } + + @Override + public boolean hasNext() { + if (next == null) { + advance(); + } + return next != null; + } + + @Override + public NitriteId next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + NitriteId id = next; + next = null; + return id; + } + + private void advance() { + if (!started) { + started = true; + key = reverse + ? (range.upperInclusive ? map.floorKey(IndexEntryKey.upperBound(range.upper)) : map.lowerKey(IndexEntryKey.lowerBound(range.upper))) + : (range.lowerInclusive ? map.ceilingKey(IndexEntryKey.lowerBound(range.lower)) : map.higherKey(IndexEntryKey.upperBound(range.lower))); + } + while (true) { + while (!group.isEmpty()) { + NitriteId id = group.pollFirst(); + if (seen.add(id)) { + next = id; + return; + } + } + if (key == null || !within(key)) { + key = null; + return; + } + if (reverse) { + // read this key's group forwards, then continue below it + DBValue value = key.getValue(); + for (IndexEntryKey k = map.ceilingKey(IndexEntryKey.lowerBound(value)); + k != null && k.getValue().compareTo(value) == 0; + k = map.higherKey(k)) { + if (map.get(k) != null) { + group.addLast(k.getNitriteId()); + } + } + key = map.lowerKey(IndexEntryKey.lowerBound(value)); + } else { + IndexEntryKey current = key; + key = map.higherKey(current); + if (map.get(current) != null) { + // removed in the current transaction otherwise; navigation still surfaces the key + group.addLast(current.getNitriteId()); + } + } + } + } + + private boolean within(IndexEntryKey candidate) { + if (reverse) { + int cmp = candidate.getValue().compareTo(range.lower); + return range.lowerInclusive ? cmp >= 0 : cmp > 0; + } + int cmp = candidate.getValue().compareTo(range.upper); + return range.upperInclusive ? cmp <= 0 : cmp < 0; + } + } + @Override @SuppressWarnings("unchecked") public List> readSortKeys(long collectionSize) { diff --git a/nitrite/src/test/java/org/dizitart/no2/collection/LazyIndexScanTest.java b/nitrite/src/test/java/org/dizitart/no2/collection/LazyIndexScanTest.java new file mode 100644 index 00000000..8ea41b0c --- /dev/null +++ b/nitrite/src/test/java/org/dizitart/no2/collection/LazyIndexScanTest.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2017-2020. Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.dizitart.no2.collection; + +import org.dizitart.no2.Nitrite; +import org.dizitart.no2.common.SortOrder; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; + +import static org.dizitart.no2.collection.FindOptions.orderBy; +import static org.dizitart.no2.collection.FindOptions.skipBy; +import static org.dizitart.no2.filters.FluentFilter.where; +import static org.dizitart.no2.index.IndexOptions.indexOptions; +import static org.dizitart.no2.index.IndexType.NON_UNIQUE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Equality and two-sided range queries on a non-unique index now stream their ids from the + * index instead of materializing every match first. The results, their order, their count and + * paging over them must be exactly what the materialized scan produced. + */ +public class LazyIndexScanTest { + private Nitrite db; + private NitriteCollection collection; + + @Before + public void setUp() { + db = Nitrite.builder().openOrCreate(); + collection = db.getCollection("lazy"); + collection.createIndex(indexOptions(NON_UNIQUE), "k"); + collection.createIndex(indexOptions(NON_UNIQUE), "tags"); + for (int i = 0; i < 200; i++) { + collection.insert(Document.createDocument("n", i).put("k", i % 10).put("tags", new String[]{"t" + (i % 3), "x"})); + } + } + + @After + public void tearDown() { + db.close(); + } + + @Test + public void testEqualityResultsCountAndPaging() { + DocumentCursor cursor = collection.find(where("k").eq(3)); + assertEquals(20, cursor.size()); + assertEquals(20, cursor.toList().size()); + assertNotNull(collection.find(where("k").eq(3)).firstOrNull()); + assertEquals(3, collection.find(where("k").eq(3), skipBy(5).limit(3)).toList().size()); + assertEquals(0, collection.find(where("k").eq(42)).size()); + } + + @Test + public void testRangeResultsInIndexOrderBothWays() { + List ascending = collection.find(where("k").between(2, 4)).toList(); + assertEquals(60, ascending.size()); + assertEquals(2, ascending.get(0).get("k", Integer.class).intValue()); + assertEquals(4, ascending.get(ascending.size() - 1).get("k", Integer.class).intValue()); + + List descending = collection.find(where("k").between(2, 4), orderBy("k", SortOrder.Descending)).toList(); + assertEquals(60, descending.size()); + assertEquals(4, descending.get(0).get("k", Integer.class).intValue()); + assertEquals(2, descending.get(descending.size() - 1).get("k", Integer.class).intValue()); + } + + @Test + public void testMultiValuedFieldReturnsEachDocumentOnce() { + assertEquals(200, collection.find(where("tags").eq("x")).size()); + assertEquals(200, collection.find(where("tags").eq("x")).toList().size()); + assertEquals(200, collection.find(where("tags").between("t0", "t9")).size()); + } + + @Test + public void testCountFollowsRemovals() { + collection.remove(where("n").eq(3)); + assertEquals(19, collection.find(where("k").eq(3)).size()); + assertEquals(19, collection.find(where("k").eq(3)).toList().size()); + } + + @Test + public void testShapesOutsideTheLazyPathAreUnchanged() { + assertEquals(40, collection.find(where("k").in(1, 2)).size()); + assertEquals(40, collection.find(where("k").gt(7)).size()); + assertEquals(20, collection.find(where("k").eq(3).and(where("n").lt(200))).size()); + } +} diff --git a/nitrite/src/test/java/org/dizitart/no2/index/SingleFieldIndexTest.java b/nitrite/src/test/java/org/dizitart/no2/index/SingleFieldIndexTest.java index 05c54b44..8223661f 100644 --- a/nitrite/src/test/java/org/dizitart/no2/index/SingleFieldIndexTest.java +++ b/nitrite/src/test/java/org/dizitart/no2/index/SingleFieldIndexTest.java @@ -24,6 +24,10 @@ import org.dizitart.no2.common.Fields; import org.dizitart.no2.filters.ComparableFilter; import org.dizitart.no2.filters.IndexScanFilter; +import org.dizitart.no2.common.RecordStream; +import org.dizitart.no2.store.NitriteStore; +import org.dizitart.no2.store.memory.InMemoryMap; +import org.dizitart.no2.filters.SortingAwareFilter; import org.dizitart.no2.store.NitriteMap; import org.dizitart.no2.store.memory.InMemoryStore; import org.junit.Test; @@ -33,12 +37,18 @@ import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; +import java.util.HashMap; import static org.dizitart.no2.common.tuples.Pair.pair; import static org.dizitart.no2.common.util.IndexUtils.deriveCompositeIndexMapName; import static org.dizitart.no2.common.util.IndexUtils.deriveIndexMapName; import static org.dizitart.no2.filters.FluentFilter.where; import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; public class SingleFieldIndexTest { @Test @@ -107,5 +117,89 @@ public void testLegacyArrayIndexMigratedToComposite() { assertFalse(store.hasMap(legacyName)); assertTrue(store.hasMap(deriveCompositeIndexMapName(desc))); } -} + @Test + public void testLazyStreamMatchesMaterializedScanForEqualityAndRange() { + InMemoryStore store = new InMemoryStore(); + IndexDescriptor desc = new IndexDescriptor(IndexType.NON_UNIQUE, Fields.withNames("k"), "c"); + SingleFieldIndex index = new SingleFieldIndex(desc, store); + for (long id = 1; id <= 30; id++) { + index.write(values(id, "k", (int) (id % 5))); // five keys, six ids each + } + index.write(values(31L, "k", new int[]{1, 2, 3})); // one document under three keys + + FindPlan eq = plan(desc, Collections.singletonList((ComparableFilter) where("k").eq(2)), null); + assertEquals(new ArrayList<>(index.findNitriteIds(eq)), index.findNitriteIdStream(eq).toList()); + + List between = Arrays.asList((ComparableFilter) where("k").gte(1), (ComparableFilter) where("k").lt(3)); + FindPlan range = plan(desc, between, null); + assertEquals(new ArrayList<>(index.findNitriteIds(range)), index.findNitriteIdStream(range).toList()); + assertEquals("id 31 is under two keys of the range but returned once", 13, index.findNitriteIdStream(range).toList().size()); + + Map descending = new HashMap<>(); + descending.put("k", true); + FindPlan reversed = plan(desc, between, descending); + assertEquals(new ArrayList<>(index.findNitriteIds(reversed)), index.findNitriteIdStream(reversed).toList()); + // key groups are visited backwards, ids inside a group keep their stored order + List forward = index.findNitriteIdStream(range).toList(); + List backward = index.findNitriteIdStream(reversed).toList(); + assertEquals(NitriteId.createId(1L), forward.get(0)); + assertEquals(NitriteId.createId(2L), backward.get(0)); + assertEquals(forward.size(), backward.size()); + } + + @Test + public void testLazyStreamDeclinesShapesItDoesNotServe() { + InMemoryStore store = new InMemoryStore(); + IndexDescriptor desc = new IndexDescriptor(IndexType.NON_UNIQUE, Fields.withNames("k"), "c"); + SingleFieldIndex index = new SingleFieldIndex(desc, store); + index.write(values(1L, "k", 1)); + + assertNull("one-sided range", index.findNitriteIdStream(plan(desc, Collections.singletonList((ComparableFilter) where("k").gt(0)), null))); + assertNull("in filter", index.findNitriteIdStream(plan(desc, Collections.singletonList((ComparableFilter) where("k").in(1, 2)), null))); + assertNull("no scan filter", index.findNitriteIdStream(new FindPlan())); + + IndexDescriptor unique = new IndexDescriptor(IndexType.UNIQUE, Fields.withNames("k"), "c"); + SingleFieldIndex uniqueIndex = new SingleFieldIndex(unique, store); + uniqueIndex.write(values(1L, "k", 1)); + assertNull("unique layout", uniqueIndex.findNitriteIdStream(plan(unique, Collections.singletonList((ComparableFilter) where("k").eq(1)), null))); + } + + @Test + public void testLazyStreamReadsOnlyAsFarAsConsumed() { + IndexDescriptor desc = new IndexDescriptor(IndexType.NON_UNIQUE, Fields.withNames("k"), "c"); + InMemoryMap composite = spy(new InMemoryMap<>(deriveCompositeIndexMapName(desc), new InMemoryStore())); + for (long id = 1; id <= 500; id++) { + composite.put(new IndexEntryKey(new DBValue("same"), NitriteId.createId(id)), Boolean.TRUE); + } + NitriteStore store = mock(NitriteStore.class); + when(store.hasMap(anyString())).thenReturn(false); + doReturn(composite).when(store).openMap(eq(deriveCompositeIndexMapName(desc)), any(), any()); + clearInvocations(composite); + + SingleFieldIndex index = new SingleFieldIndex(desc, store); + FindPlan eq = plan(desc, Collections.singletonList((ComparableFilter) where("k").eq("same")), null); + RecordStream stream = index.findNitriteIdStream(eq); + assertNotNull(stream); + + assertEquals(NitriteId.createId(1L), stream.iterator().next()); + verify(composite, atMost(2)).higherKey(any()); + verify(composite, never()).entries(); + assertEquals(500, stream.toList().size()); + } + + private static FindPlan plan(IndexDescriptor desc, List filters, Map scanOrder) { + FindPlan plan = new FindPlan(); + plan.setIndexDescriptor(desc); + plan.setIndexScanFilter(new IndexScanFilter(filters)); + plan.setIndexScanOrder(scanOrder); + return plan; + } + + private static FieldValues values(long id, String field, Object value) { + FieldValues fieldValues = new FieldValues(); + fieldValues.setNitriteId(NitriteId.createId(id)); + fieldValues.getValues().add(pair(field, value)); + return fieldValues; + } +}