From 502faf224ba25a89bd53fb4de9793d5bbb2b2692 Mon Sep 17 00:00:00 2001 From: vinayak hegde Date: Wed, 26 Jun 2024 20:45:32 +0530 Subject: [PATCH] SOLR-16677: Update Solr to use new Lucene 9.5 storedFields() API (#1557) Co-authored-by: Christine Poerschke Co-authored-by: Michael Gibney --- solr/CHANGES.txt | 4 + .../apache/solr/core/QuerySenderListener.java | 11 +- .../org/apache/solr/handler/BlobHandler.java | 4 +- .../solr/handler/MoreLikeThisHandler.java | 7 +- .../handler/admin/IndexSizeEstimator.java | 8 +- .../handler/admin/LukeRequestHandler.java | 11 +- .../component/MoreLikeThisComponent.java | 9 +- .../component/RealTimeGetComponent.java | 4 +- .../component/ResponseLogComponent.java | 7 +- .../component/TermVectorComponent.java | 6 +- .../highlight/DefaultSolrHighlighter.java | 56 ++++---- .../solr/highlight/SolrHighlighter.java | 7 +- .../highlight/UnifiedSolrHighlighter.java | 7 +- .../index/SlowCompositeReaderWrapper.java | 2 + .../apache/solr/response/DocsStreamer.java | 2 +- .../apache/solr/response/ResultContext.java | 11 ++ .../transform/ChildDocTransformer.java | 4 +- .../org/apache/solr/search/ReturnFields.java | 2 +- .../solr/search/SolrDocumentFetcher.java | 124 +++++++++++------- .../apache/solr/search/SolrIndexSearcher.java | 15 ++- .../TopGroupsResultTransformer.java | 12 +- .../org/apache/solr/util/SolrPluginUtils.java | 7 +- .../apache/solr/schema/TestPointFields.java | 4 +- .../org/apache/solr/search/TestDocSet.java | 2 + .../apache/solr/search/TestStressLucene.java | 2 +- .../solr/search/function/TestOrdValues.java | 4 +- .../TestFieldCacheVsDocValues.java | 4 +- .../clustering/ClusteringComponent.java | 4 +- .../solr/ltr/feature/FieldValueFeature.java | 8 +- 29 files changed, 233 insertions(+), 115 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 2ca207bbf42..26a88337aac 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -146,6 +146,10 @@ Optimizations * SOLR-17330: When not set, loadOnStartup defaults to true, which is the default choice for a core. (Pierre Salagnac via Eric Pugh) +* SOLR-16677: Update Solr to use new Lucene 9.5 storedFields() API. This removes the use of ThreadLocal for + stored field state, reducing heap usage especially for high-core-count, high-field-count, high-thread-count + cases (Vinayak Hegde, Christine Poerschke, Kevin Risden, David Smiley, Michael Gibney) + Bug Fixes --------------------- * SOLR-12813: subqueries should respect basic auth. (Rudy Seitz via Eric Pugh) diff --git a/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java b/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java index 30c310ca0e3..a612b27809a 100644 --- a/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java +++ b/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java @@ -30,6 +30,7 @@ import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,13 +81,19 @@ public void close() {} NamedList values = rsp.getValues(); for (int i = 0; i < values.size(); i++) { Object o = values.getVal(i); + SolrDocumentFetcher docFetcher = null; if (o instanceof ResultContext) { - o = ((ResultContext) o).getDocList(); + ResultContext ctx = (ResultContext) o; + o = ctx.getDocList(); + docFetcher = ctx.getDocFetcher(); } if (o instanceof DocList) { DocList docs = (DocList) o; + if (docFetcher == null) { + docFetcher = newSearcher.getDocFetcher(); + } for (DocIterator iter = docs.iterator(); iter.hasNext(); ) { - newSearcher.doc(iter.nextDoc()); + docFetcher.doc(iter.nextDoc()); } } } diff --git a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java index 3512b0154ed..ff59b99f624 100644 --- a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java @@ -140,7 +140,7 @@ public void handleRequestBody(final SolrQueryRequest req, SolrQueryResponse rsp) long version = 0; if (docs.totalHits.value > 0) { - Document doc = req.getSearcher().doc(docs.scoreDocs[0].doc); + Document doc = req.getSearcher().getDocFetcher().doc(docs.scoreDocs[0].doc); Number n = doc.getField("version").numericValue(); version = n.longValue(); } @@ -214,7 +214,7 @@ public void handleRequestBody(final SolrQueryRequest req, SolrQueryResponse rsp) @Override public void write(OutputStream os) throws IOException { - Document doc = req.getSearcher().doc(docs.scoreDocs[0].doc); + Document doc = req.getSearcher().getDocFetcher().doc(docs.scoreDocs[0].doc); IndexableField sf = doc.getField("blob"); FieldType fieldType = req.getSchema().getField("blob").getType(); ByteBuffer buf = (ByteBuffer) fieldType.toObject(sf); diff --git a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java index 2027012d23a..8b4a292b0a1 100644 --- a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java @@ -26,9 +26,11 @@ import java.util.List; import java.util.Map; import java.util.regex.Pattern; +import net.jcip.annotations.NotThreadSafe; import org.apache.lucene.document.Document; import org.apache.lucene.index.ExitableDirectoryReader; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; import org.apache.lucene.queries.mlt.MoreLikeThis; import org.apache.lucene.search.BooleanClause; @@ -282,10 +284,12 @@ public static class InterestingTerm { } /** Helper class for MoreLikeThis that can be called from other request handlers */ + @NotThreadSafe public static class MoreLikeThisHelper { final SolrIndexSearcher searcher; final MoreLikeThis mlt; final IndexReader reader; + final StoredFields storedFields; final SchemaField uniqueKeyField; final boolean needDocSet; Map boostFields; @@ -293,6 +297,7 @@ public static class MoreLikeThisHelper { public MoreLikeThisHelper(SolrParams params, SolrIndexSearcher searcher) throws IOException { this.searcher = searcher; this.reader = searcher.getIndexReader(); + this.storedFields = this.reader.storedFields(); this.uniqueKeyField = searcher.getSchema().getUniqueKeyField(); this.needDocSet = params.getBool(FacetParams.FACET, false); @@ -394,7 +399,7 @@ private BooleanQuery getBoostedQuery(Query mltquery) { public DocListAndSet getMoreLikeThis( int id, int start, int rows, List filters, int flags) throws IOException { - Document doc = reader.document(id); + Document doc = this.storedFields.document(id); final Query boostedQuery = getBoostedMLTQuery(id); // exclude current document from results diff --git a/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java b/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java index b4f222a7a49..87d4764acba 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java @@ -45,6 +45,8 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StandardDirectoryReader; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.StoredFields; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.DocIdSetIterator; @@ -384,11 +386,12 @@ private void estimateTermVectors(Map result) throws IOException for (LeafReaderContext leafReaderContext : reader.leaves()) { LeafReader leafReader = leafReaderContext.reader(); Bits liveDocs = leafReader.getLiveDocs(); + TermVectors leafTermVectors = leafReader.termVectors(); for (int docId = 0; docId < leafReader.maxDoc(); docId += samplingStep) { if (liveDocs != null && !liveDocs.get(docId)) { continue; } - Fields termVectors = leafReader.getTermVectors(docId); + Fields termVectors = leafTermVectors.get(docId); if (termVectors == null) { continue; } @@ -608,11 +611,12 @@ private void estimateStoredFields(Map result) throws IOException mergeInstance.close(); } } else { + StoredFields storedFields = leafReader.storedFields(); for (int docId = 0; docId < leafReader.maxDoc(); docId += samplingStep) { if (liveDocs != null && !liveDocs.get(docId)) { continue; } - leafReader.document(docId, visitor); + storedFields.document(docId, visitor); } } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java index 318f6d0842b..c99c6f9afea 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java @@ -52,7 +52,9 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.MultiTerms; import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.DocIdSetIterator; @@ -161,7 +163,7 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw } Document doc = null; try { - doc = reader.document(docId); + doc = reader.storedFields().document(docId); } catch (Exception ex) { } if (doc == null) { @@ -316,6 +318,7 @@ private static SimpleOrderedMap getDocumentFieldsInfo( Document doc, int docId, IndexReader reader, IndexSchema schema) throws IOException { final CharsRefBuilder spare = new CharsRefBuilder(); SimpleOrderedMap finfo = new SimpleOrderedMap<>(); + TermVectors termVectors = null; for (Object o : doc.getFields()) { Field field = (Field) o; SimpleOrderedMap f = new SimpleOrderedMap<>(); @@ -353,7 +356,8 @@ private static SimpleOrderedMap getDocumentFieldsInfo( // If we have a term vector, return that if (field.fieldType().storeTermVectors()) { try { - Terms v = reader.getTermVector(docId, field.name()); + if (termVectors == null) termVectors = reader.termVectors(); + Terms v = termVectors.get(docId, field.name()); if (v != null) { SimpleOrderedMap tfv = new SimpleOrderedMap<>(); final TermsEnum termsEnum = v.iterator(); @@ -462,6 +466,7 @@ private static Document getFirstLiveDoc(Terms terms, LeafReader reader) throws I PostingsEnum postingsEnum = null; TermsEnum termsEnum = terms.iterator(); BytesRef text; + StoredFields storedFields = reader.storedFields(); // Deal with the chance that the first bunch of terms are in deleted documents. Is there a // better way? for (int idx = 0; idx < 1000 && postingsEnum == null; ++idx) { @@ -476,7 +481,7 @@ private static Document getFirstLiveDoc(Terms terms, LeafReader reader) throws I if (liveDocs != null && liveDocs.get(postingsEnum.docID())) { continue; } - return reader.document(postingsEnum.docID()); + return storedFields.document(postingsEnum.docID()); } } return null; diff --git a/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java b/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java index bc1267bfd54..2b41bdd1245 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java @@ -49,6 +49,7 @@ import org.apache.solr.search.DocListAndSet; import org.apache.solr.search.QueryLimits; import org.apache.solr.search.ReturnFields; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrReturnFields; import org.slf4j.Logger; @@ -103,6 +104,7 @@ public void process(ResponseBuilder rb) throws IOException { new MoreLikeThisHandler.MoreLikeThisHelper(params, searcher); NamedList> mltQueryByDocKey = new NamedList<>(); QueryLimits queryLimits = QueryLimits.getCurrentLimits(); + SolrDocumentFetcher docFetcher = rb.req.getSearcher().getDocFetcher(); for (DocIterator results = rb.getResults().docList.iterator(); results.hasNext(); ) { int docId = results.nextDoc(); final List interestingTerms = @@ -114,7 +116,7 @@ public void process(ResponseBuilder rb) throws IOException { break; } final String uniqueKey = rb.req.getSchema().getUniqueKeyField().getName(); - final Document document = rb.req.getSearcher().doc(docId); + final Document document = docFetcher.doc(docId); final String uniqueVal = rb.req.getSchema().printableUniqueKey(document); final NamedList mltQ = mltViaQueryParams(rb.req.getSchema(), interestingTerms, uniqueKey, uniqueVal); @@ -423,12 +425,13 @@ NamedList getMoreLikeThese( QueryLimits queryLimits = QueryLimits.getCurrentLimits(); + SolrDocumentFetcher docFetcher = searcher.getDocFetcher(); while (iterator.hasNext()) { int id = iterator.nextDoc(); int rows = p.getInt(MoreLikeThisParams.DOC_COUNT, 5); DocListAndSet similarDocuments = mltHelper.getMoreLikeThis(id, 0, rows, null, flags); - String name = schema.printableUniqueKey(searcher.doc(id)); + String name = schema.printableUniqueKey(docFetcher.doc(id)); mltResponse.add(name, similarDocuments.docList); if (dbg != null) { @@ -440,7 +443,7 @@ NamedList getMoreLikeThese( DocIterator similarDocumentsIterator = similarDocuments.docList.iterator(); while (similarDocumentsIterator.hasNext()) { int mltid = similarDocumentsIterator.nextDoc(); - String key = schema.printableUniqueKey(searcher.doc(mltid)); + String key = schema.printableUniqueKey(docFetcher.doc(mltid)); explains.add(key, searcher.explain(mltHelper.getRealMLTQuery(), mltid)); } docDbg.add("explain", explains); diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java index 33fa334758a..f1fc77d9ceb 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java @@ -347,10 +347,10 @@ public void process(ResponseBuilder rb) throws IOException { if (docid < 0) continue; + SolrDocumentFetcher docFetcher = searcherInfo.getSearcher().getDocFetcher(); Document luceneDocument = - searcherInfo.getSearcher().doc(docid, rsp.getReturnFields().getLuceneFieldNames()); + docFetcher.doc(docid, rsp.getReturnFields().getLuceneFieldNames()); SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema()); - SolrDocumentFetcher docFetcher = searcherInfo.getSearcher().getDocFetcher(); if (reuseDvIters == null) { reuseDvIters = new DocValuesIteratorCache(searcherInfo.getSearcher()); } diff --git a/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java b/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java index eba622615a3..6d8ee1957fb 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java @@ -24,6 +24,7 @@ import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; /** @@ -90,9 +91,10 @@ protected void processIds( StringBuilder sb = new StringBuilder(); Set fields = Collections.singleton(schema.getUniqueKeyField().getName()); + SolrDocumentFetcher docFetcher = searcher.getDocFetcher(); for (DocIterator iter = dl.iterator(); iter.hasNext(); ) { - sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields))).append(','); + sb.append(schema.printableUniqueKey(docFetcher.doc(iter.nextDoc(), fields))).append(','); } if (sb.length() > 0) { rb.rsp.addToLog("responseLog", sb.substring(0, sb.length() - 1)); @@ -105,8 +107,9 @@ protected void processScores( StringBuilder sb = new StringBuilder(); Set fields = Collections.singleton(schema.getUniqueKeyField().getName()); + SolrDocumentFetcher docFetcher = searcher.getDocFetcher(); for (DocIterator iter = dl.iterator(); iter.hasNext(); ) { - sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields))) + sb.append(schema.printableUniqueKey(docFetcher.doc(iter.nextDoc(), fields))) .append(':') .append(iter.score()) .append(','); diff --git a/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java b/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java index 6373b0a6337..73eaa3b817b 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java @@ -34,6 +34,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; @@ -251,6 +252,7 @@ public void process(ResponseBuilder rb) throws IOException { SolrIndexSearcher searcher = rb.req.getSearcher(); IndexReader reader = searcher.getIndexReader(); + TermVectors termVectors = reader.termVectors(); // the TVMapper is a TermVectorMapper which can be used to optimize loading of Term Vectors // Only load the id field to get the uniqueKey of that @@ -277,7 +279,7 @@ public void process(ResponseBuilder rb) throws IOException { if (null != fields) { for (Map.Entry entry : fieldOptions.entrySet()) { final String field = entry.getKey(); - final Terms vector = reader.getTermVector(docId, field); + final Terms vector = termVectors.get(docId, field); if (vector != null) { TermsEnum termsEnum = vector.iterator(); mapOneVector(docNL, entry.getValue(), reader, docId, termsEnum, field); @@ -285,7 +287,7 @@ public void process(ResponseBuilder rb) throws IOException { } } else { // extract all fields - final Fields vectors = reader.getTermVectors(docId); + final Fields vectors = termVectors.get(docId); // There can be no documents with vectors if (vectors != null) { for (String field : vectors) { diff --git a/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java index 6fc2d475942..ba22b2e317c 100644 --- a/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java +++ b/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java @@ -33,10 +33,9 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.index.Fields; -import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -78,6 +77,7 @@ import org.apache.solr.schema.SchemaField; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrReturnFields; import org.apache.solr.util.plugin.PluginInfoInitialized; @@ -482,15 +482,16 @@ public NamedList doHighlighting( // Lazy container for fvh and fieldQuery FvhContainer fvhContainer = new FvhContainer(null, null); - IndexReader reader = - new TermVectorReusingLeafReader(req.getSearcher().getSlowAtomicReader()); // SOLR-5855 + IndexReader reader = req.getSearcher().getSlowAtomicReader(); + TermVectors termVectors = new ReusingTermVectors(reader); // SOLR-5855 // Highlight each document NamedList fragments = new SimpleOrderedMap<>(); + SolrDocumentFetcher docFetcher = searcher.getDocFetcher(); DocIterator iterator = docs.iterator(); for (int i = 0; i < docs.size(); i++) { int docId = iterator.nextDoc(); - SolrDocument doc = searcher.getDocFetcher().solrDoc(docId, returnFields); + SolrDocument doc = docFetcher.solrDoc(docId, returnFields); NamedList docHighlights = new SimpleOrderedMap<>(); // Highlight per-field @@ -500,10 +501,11 @@ public NamedList doHighlighting( Object fieldHighlights; // object type allows flexibility for subclassers fieldHighlights = doHighlightingOfField( - doc, docId, schemaField, fvhContainer, query, reader, req, params); + doc, docId, schemaField, fvhContainer, query, reader, termVectors, req, params); if (fieldHighlights == null) { - fieldHighlights = alternateField(doc, docId, fieldName, fvhContainer, query, reader, req); + fieldHighlights = + alternateField(doc, docId, fieldName, fvhContainer, query, reader, termVectors, req); } if (fieldHighlights != null) { @@ -522,6 +524,7 @@ protected Object doHighlightingOfField( FvhContainer fvhContainer, Query query, IndexReader reader, + TermVectors termVectors, SolrQueryRequest req, SolrParams params) throws IOException { @@ -570,7 +573,8 @@ protected void flatten( fieldHighlights = doHighlightingByFastVectorHighlighter(doc, docId, schemaField, fvhContainer, reader, req); } else { // standard/default highlighter - fieldHighlights = doHighlightingByHighlighter(doc, docId, schemaField, query, reader, req); + fieldHighlights = + doHighlightingByHighlighter(doc, docId, schemaField, query, termVectors, req); } return fieldHighlights; } @@ -656,7 +660,7 @@ protected Object doHighlightingByHighlighter( int docId, SchemaField schemaField, Query query, - IndexReader reader, + TermVectors termVectors, SolrQueryRequest req) throws IOException { final SolrParams params = req.getParams(); @@ -696,7 +700,7 @@ protected Object doHighlightingByHighlighter( // Try term vectors, which is faster // note: offsets are minimally sufficient for this HL. - final Fields tvFields = schemaField.storeTermOffsets() ? reader.getTermVectors(docId) : null; + final Fields tvFields = schemaField.storeTermOffsets() ? termVectors.get(docId) : null; final TokenStream tvStream = TokenSources.getTermVectorTokenStreamOrNull(fieldName, tvFields, maxCharsToAnalyze - 1); // We need to wrap in OffsetWindowTokenFilter if multi-valued @@ -858,6 +862,7 @@ protected Object alternateField( FvhContainer fvhContainer, Query query, IndexReader reader, + TermVectors termVectors, SolrQueryRequest req) throws IOException { IndexSchema schema = req.getSearcher().getSchema(); @@ -887,7 +892,7 @@ protected Object alternateField( req.setParams(SolrParams.wrapDefaults(new MapSolrParams(invariants), origParams)); fieldHighlights = doHighlightingOfField( - doc, docId, schemaField, fvhContainer, query, reader, req, params); + doc, docId, schemaField, fvhContainer, query, reader, termVectors, req, params); req.setParams(origParams); if (fieldHighlights != null) { return fieldHighlights; @@ -1087,35 +1092,30 @@ public boolean incrementToken() throws IOException { } /** - * Wraps a DirectoryReader that caches the {@link LeafReader#getTermVectors(int)} so that if the - * next call has the same ID, then it is reused. + * Wraps a TermVectors and caches the {@link TermVectors#get(int)} so that if the next call has + * the same ID, then it is reused. */ - static class TermVectorReusingLeafReader extends FilterLeafReader { + static class ReusingTermVectors extends TermVectors { + private final IndexReader reader; + private TermVectors in; private int lastDocId = -1; private Fields tvFields; - public TermVectorReusingLeafReader(LeafReader in) { - super(in); + public ReusingTermVectors(IndexReader reader) { + this.reader = reader; } @Override - public Fields getTermVectors(int docID) throws IOException { + public Fields get(int docID) throws IOException { if (docID != lastDocId) { + if (in == null) { + in = reader.termVectors(); + } lastDocId = docID; - tvFields = in.getTermVectors(docID); + tvFields = in.get(docID); } return tvFields; } - - @Override - public CacheHelper getCoreCacheHelper() { - return null; - } - - @Override - public CacheHelper getReaderCacheHelper() { - return null; - } } } diff --git a/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java index 4e7729fa49a..e106c471f63 100644 --- a/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java +++ b/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java @@ -29,6 +29,7 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.util.SolrPluginUtils; public abstract class SolrHighlighter { @@ -73,7 +74,11 @@ public String[] getHighlightFields( } else { fields = expandWildcardsInFields( - () -> request.getSearcher().getDocFetcher().getStoredHighlightFieldNames(), fields); + () -> + request + .getSearcher() + .interrogateDocFetcher(SolrDocumentFetcher::getStoredHighlightFieldNames), + fields); } // Trim them now in case they haven't been yet. Not needed for all code-paths above but do it diff --git a/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java index f14c43d1fd3..4390893ec76 100644 --- a/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java +++ b/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java @@ -49,6 +49,7 @@ import org.apache.solr.schema.SchemaField; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrReturnFields; import org.apache.solr.util.RTimerTree; @@ -223,10 +224,11 @@ protected String[] getUniqueKeys(SolrIndexSearcher searcher, int[] docIDs) throw SchemaField keyField = schema.getUniqueKeyField(); if (keyField != null) { SolrReturnFields returnFields = new SolrReturnFields(keyField.getName(), null); + SolrDocumentFetcher docFetcher = searcher.getDocFetcher(); String[] uniqueKeys = new String[docIDs.length]; for (int i = 0; i < docIDs.length; i++) { int docid = docIDs[i]; - SolrDocument solrDoc = searcher.getDocFetcher().solrDoc(docid, returnFields); + SolrDocument solrDoc = docFetcher.solrDoc(docid, returnFields); uniqueKeys[i] = schema.printableUniqueKey(solrDoc); } return uniqueKeys; @@ -446,7 +448,8 @@ protected Predicate getFieldMatcher(String field) { if (queryFieldPattern != null && queryFieldPattern.length != 0) { Supplier> indexedFieldsSupplier = - () -> solrIndexSearcher.getDocFetcher().getIndexedFieldNames(); + () -> + solrIndexSearcher.interrogateDocFetcher(SolrDocumentFetcher::getIndexedFieldNames); Set fields = Set.of(expandWildcardsInFields(indexedFieldsSupplier, queryFieldPattern)); diff --git a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java index f8fffe9bce8..cdfbb181671 100644 --- a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java +++ b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java @@ -313,6 +313,7 @@ public NumericDocValues getNormValues(String field) throws IOException { } @Override + @Deprecated public Fields getTermVectors(int docID) throws IOException { return in.getTermVectors(docID); } @@ -342,6 +343,7 @@ public int maxDoc() { } @Override + @Deprecated public void document(int docID, StoredFieldVisitor visitor) throws IOException { ensureOpen(); in.document(docID, visitor); diff --git a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java index 8fdcefa3d3f..485d9c9a1e8 100644 --- a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java +++ b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java @@ -73,7 +73,7 @@ public DocsStreamer(ResultContext rctx) { this.docs = rctx.getDocList(); transformer = rctx.getReturnFields().getTransformer(); docIterator = this.docs.iterator(); - docFetcher = rctx.getSearcher().getDocFetcher(); + docFetcher = rctx.getDocFetcher(); solrReturnFields = (SolrReturnFields) rctx.getReturnFields(); if (transformer != null) transformer.setContext(rctx); diff --git a/solr/core/src/java/org/apache/solr/response/ResultContext.java b/solr/core/src/java/org/apache/solr/response/ResultContext.java index a74d0484d08..ea07d076da5 100644 --- a/solr/core/src/java/org/apache/solr/response/ResultContext.java +++ b/solr/core/src/java/org/apache/solr/response/ResultContext.java @@ -23,17 +23,28 @@ import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.search.DocList; import org.apache.solr.search.ReturnFields; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; /** A class to hold the QueryResult and the Query */ public abstract class ResultContext { + private SolrDocumentFetcher docFetcher; + public abstract DocList getDocList(); public abstract ReturnFields getReturnFields(); public abstract SolrIndexSearcher getSearcher(); + public SolrDocumentFetcher getDocFetcher() { + if (docFetcher == null) { + return docFetcher = getSearcher().getDocFetcher(); + } else { + return docFetcher; + } + } + public abstract Query getQuery(); // TODO: any reason to allow for retrieval of any filters as well? diff --git a/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java b/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java index 7eaac00e52c..d8a4f084226 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java +++ b/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java @@ -45,6 +45,7 @@ import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.BitsFilteredPostingsEnum; import org.apache.solr.search.DocSet; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrReturnFields; import org.slf4j.Logger; @@ -173,6 +174,7 @@ public void transform(SolrDocument rootDoc, int rootDocId) { final Map>> pendingParentPathsToChildren = new HashMap<>(); + SolrDocumentFetcher docFetcher = context.getDocFetcher(); final int firstChildId = segBaseId + segPrevRootId + 1; int matches = 0; // Loop each child ID up to the parent (exclusive). @@ -207,7 +209,7 @@ public void transform(SolrDocument rootDoc, int rootDocId) { ++matches; // note: includes ancestors that are not necessarily in childDocSet // load the doc - SolrDocument doc = searcher.getDocFetcher().solrDoc(docId, childReturnFields); + SolrDocument doc = docFetcher.solrDoc(docId, childReturnFields); if (childReturnFields.getTransformer() != null) { if (childReturnFields.getTransformer().context == null) { childReturnFields.getTransformer().setContext(context); diff --git a/solr/core/src/java/org/apache/solr/search/ReturnFields.java b/solr/core/src/java/org/apache/solr/search/ReturnFields.java index e93b0fae772..44dcb12491e 100644 --- a/solr/core/src/java/org/apache/solr/search/ReturnFields.java +++ b/solr/core/src/java/org/apache/solr/search/ReturnFields.java @@ -28,7 +28,7 @@ public abstract class ReturnFields { /** * Set of field names with their exact names from the lucene index. Class such as ResponseWriters - * pass this to {@link SolrIndexSearcher#doc(int, Set)}. + * pass this to {@link SolrDocumentFetcher#doc(int, Set)}. * *

NOTE: In some situations, this method may return null even if {@link * #wantsAllFields()} is false. For example: When glob expressions are used ({@link diff --git a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java index e6ccb9edd18..ea7cf854aab 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.Reader; +import java.io.UncheckedIOException; import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -42,7 +43,6 @@ import org.apache.lucene.document.StoredValue; import org.apache.lucene.document.TextField; import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; @@ -57,6 +57,7 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.misc.document.LazyDocument; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; @@ -114,11 +115,38 @@ public class SolrDocumentFetcher { private final Set largeFields; - private Collection storedHighlightFieldNames; // lazy populated; use getter + private final Collection[] storedHighlightFieldNames; // lazy populated; use getter + + private final Collection[] indexedFieldNames; // lazy populated; use getter + + private final StoredFields storedFields; + + private SolrDocumentFetcher(SolrDocumentFetcher template, StoredFields storedFields) { + this.searcher = template.searcher; + this.nLeaves = template.nLeaves; + this.enableLazyFieldLoading = template.enableLazyFieldLoading; + this.documentCache = template.documentCache; + this.nonStoredDVsUsedAsStored = template.nonStoredDVsUsedAsStored; + this.allNonStoredDVs = template.allNonStoredDVs; + this.nonStoredDVsWithoutCopyTargets = template.nonStoredDVsWithoutCopyTargets; + this.largeFields = template.largeFields; + this.dvsCanSubstituteStored = template.dvsCanSubstituteStored; + this.allStored = template.allStored; + this.storedHighlightFieldNames = template.indexedFieldNames; + this.indexedFieldNames = template.indexedFieldNames; + this.storedFields = storedFields; + } - private Collection indexedFieldNames; // lazy populated; use getter + @Override + protected SolrDocumentFetcher clone() { + try { + return new SolrDocumentFetcher(this, searcher.getIndexReader().storedFields()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } - @SuppressWarnings({"unchecked"}) + @SuppressWarnings({"unchecked", "rawtypes"}) SolrDocumentFetcher(SolrIndexSearcher searcher, SolrConfig solrConfig, boolean cachingEnabled) { this.searcher = searcher; this.nLeaves = searcher.getTopReaderContext().leaves().size(); @@ -172,6 +200,9 @@ public class SolrDocumentFetcher { this.largeFields = Collections.unmodifiableSet(storedLargeFields); this.dvsCanSubstituteStored = Collections.unmodifiableSet(dvsCanSubstituteStored); this.allStored = Collections.unmodifiableSet(allStoreds); + this.storedFields = null; // template docFetcher should throw NPE if used directly + this.storedHighlightFieldNames = new Collection[1]; + this.indexedFieldNames = new Collection[1]; } // Does this field have both stored=true and docValues=true and is otherwise @@ -203,9 +234,9 @@ public SolrCache getDocumentCache() { * reader knows about. */ public Collection getStoredHighlightFieldNames() { - synchronized (this) { - if (storedHighlightFieldNames == null) { - storedHighlightFieldNames = new ArrayList<>(); + synchronized (storedHighlightFieldNames) { + if (storedHighlightFieldNames[0] == null) { + Collection storedHighlightFieldNames = new ArrayList<>(); for (FieldInfo fieldInfo : searcher.getFieldInfos()) { final String fieldName = fieldInfo.name; try { @@ -220,23 +251,25 @@ public Collection getStoredHighlightFieldNames() { log.warn("Field [{}] found in index, but not defined in schema.", fieldName); } } + this.storedHighlightFieldNames[0] = storedHighlightFieldNames; } - return storedHighlightFieldNames; + return storedHighlightFieldNames[0]; } } /** Returns a collection of the names of all indexed fields which the index reader knows about. */ public Collection getIndexedFieldNames() { - synchronized (this) { - if (indexedFieldNames == null) { - indexedFieldNames = new ArrayList<>(); + synchronized (indexedFieldNames) { + if (indexedFieldNames[0] == null) { + Collection indexedFieldNames = new ArrayList<>(); for (FieldInfo fieldInfo : searcher.getFieldInfos()) { if (fieldInfo.getIndexOptions() != IndexOptions.NONE) { indexedFieldNames.add(fieldInfo.name); } } + this.indexedFieldNames[0] = indexedFieldNames; } - return indexedFieldNames; + return indexedFieldNames[0]; } } @@ -272,10 +305,9 @@ public Document doc(int i, Set fields) throws IOException { } private Document docNC(int i, Set fields) throws IOException { - final DirectoryReader reader = searcher.getIndexReader(); final SolrDocumentStoredFieldVisitor visitor = - new SolrDocumentStoredFieldVisitor(fields, reader, i); - reader.document(i, visitor); + new SolrDocumentStoredFieldVisitor(fields, searcher.getIndexReader(), i); + storedFields.document(i, visitor); return visitor.getDocument(); } @@ -368,16 +400,14 @@ public Status needsField(FieldInfo fieldInfo) throws IOException { } } - /** - * @see SolrIndexSearcher#doc(int, StoredFieldVisitor) - */ + /** Visit a document's fields using a {@link StoredFieldVisitor}. */ public void doc(int docId, StoredFieldVisitor visitor) throws IOException { if (documentCache != null) { // get cached document or retrieve it including all fields (and cache it) Document cached = doc(docId); visitFromCached(cached, visitor); } else { - searcher.getIndexReader().document(docId, visitor); + storedFields.document(docId, visitor); } } @@ -493,35 +523,33 @@ synchronized BytesRef readBytes() throws IOException { return cachedBytes; } else { BytesRef bytesRef = new BytesRef(); - searcher - .getIndexReader() - .document( - docId, - new StoredFieldVisitor() { - boolean done = false; - - @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { - if (done) { - return Status.STOP; - } - return fieldInfo.name.equals(name()) ? Status.YES : Status.NO; - } - - @Override - public void stringField(FieldInfo fieldInfo, String value) throws IOException { - Objects.requireNonNull(value, "String value should not be null"); - bytesRef.bytes = value.getBytes(StandardCharsets.UTF_8); - bytesRef.length = bytesRef.bytes.length; - done = true; - } - - @Override - public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { - throw new UnsupportedOperationException( - "'large' binary fields are not (yet) supported"); - } - }); + storedFields.document( + docId, + new StoredFieldVisitor() { + boolean done = false; + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + if (done) { + return Status.STOP; + } + return fieldInfo.name.equals(name()) ? Status.YES : Status.NO; + } + + @Override + public void stringField(FieldInfo fieldInfo, String value) throws IOException { + Objects.requireNonNull(value, "String value should not be null"); + bytesRef.bytes = value.getBytes(StandardCharsets.UTF_8); + bytesRef.length = bytesRef.bytes.length; + done = true; + } + + @Override + public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { + throw new UnsupportedOperationException( + "'large' binary fields are not (yet) supported"); + } + }); if (bytesRef.length < largeValueLengthCacheThreshold) { return cachedBytes = bytesRef; } else { diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index d4d1f64bcda..a8bc38b5699 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -35,6 +35,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.LongAdder; +import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -427,7 +428,16 @@ public SolrIndexSearcher( } public SolrDocumentFetcher getDocFetcher() { - return docFetcher; + return docFetcher.clone(); + } + + /** + * Allows interrogation of {@link #docFetcher} template (checking field names, etc.) without + * forcing it to be cloned (as it would be if an instance were retrieved via {@link + * #getDocFetcher()}). + */ + public T interrogateDocFetcher(Function func) { + return func.apply(docFetcher); } public StatsCache getStatsCache() { @@ -755,6 +765,7 @@ public LimitExceededFromScorerException(String msg) { * @see SolrDocumentFetcher */ @Override + @Deprecated public Document doc(int docId) throws IOException { return doc(docId, (Set) null); } @@ -767,6 +778,7 @@ public Document doc(int docId) throws IOException { * @see SolrDocumentFetcher */ @Override + @Deprecated public final void doc(int docId, StoredFieldVisitor visitor) throws IOException { getDocFetcher().doc(docId, visitor); } @@ -780,6 +792,7 @@ public final void doc(int docId, StoredFieldVisitor visitor) throws IOException * @see SolrDocumentFetcher */ @Override + @Deprecated public final Document doc(int i, Set fields) throws IOException { return getDocFetcher().doc(i, fields); } diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java index e385fcb941d..83255675fdd 100644 --- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java +++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java @@ -43,6 +43,7 @@ import org.apache.solr.schema.FieldType; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.grouping.Command; import org.apache.solr.search.grouping.distributed.command.QueryCommand; import org.apache.solr.search.grouping.distributed.command.QueryCommandResult; @@ -229,12 +230,13 @@ protected NamedList serializeTopGroups(TopGroups data, SchemaF groupResult.add("maxScore", searchGroup.maxScore); } + SolrDocumentFetcher docFetcher = rb.req.getSearcher().getDocFetcher(); List> documents = new ArrayList<>(); for (int i = 0; i < searchGroup.scoreDocs.length; i++) { NamedList document = new NamedList<>(); documents.add(document); - Document doc = retrieveDocument(uniqueField, searchGroup.scoreDocs[i].doc); + Document doc = retrieveDocument(uniqueField, searchGroup.scoreDocs[i].doc, docFetcher); document.add(ID, uniqueField.getType().toExternal(doc.getField(uniqueField.getName()))); if (!Float.isNaN(searchGroup.scoreDocs[i].score)) { document.add("score", searchGroup.scoreDocs[i].score); @@ -290,13 +292,14 @@ protected NamedList serializeTopDocs(QueryCommandResult result) throws I List> documents = new ArrayList<>(); queryResult.add("documents", documents); + SolrDocumentFetcher docFetcher = rb.req.getSearcher().getDocFetcher(); final IndexSchema schema = rb.req.getSearcher().getSchema(); SchemaField uniqueField = schema.getUniqueKeyField(); for (ScoreDoc scoreDoc : result.getTopDocs().scoreDocs) { NamedList document = new NamedList<>(); documents.add(document); - Document doc = retrieveDocument(uniqueField, scoreDoc.doc); + Document doc = retrieveDocument(uniqueField, scoreDoc.doc, docFetcher); document.add(ID, uniqueField.getType().toExternal(doc.getField(uniqueField.getName()))); if (!Float.isNaN(scoreDoc.score)) { document.add("score", scoreDoc.score); @@ -322,7 +325,8 @@ protected NamedList serializeTopDocs(QueryCommandResult result) throws I return queryResult; } - private Document retrieveDocument(final SchemaField uniqueField, int doc) throws IOException { - return rb.req.getSearcher().doc(doc, Collections.singleton(uniqueField.getName())); + private Document retrieveDocument( + final SchemaField uniqueField, int doc, SolrDocumentFetcher docFetcher) throws IOException { + return docFetcher.doc(doc, Collections.singleton(uniqueField.getName())); } } diff --git a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java index 3b564972eb7..a76ebe237a3 100644 --- a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java +++ b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java @@ -76,6 +76,7 @@ import org.apache.solr.search.QParser; import org.apache.solr.search.QueryParsing; import org.apache.solr.search.ReturnFields; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrQueryParser; import org.apache.solr.search.SortSpecParsing; @@ -249,9 +250,10 @@ public static void optimizePreFetchDocs( } // get documents + SolrDocumentFetcher docFetcher = searcher.getDocFetcher(); DocIterator iter = docs.iterator(); for (int i = 0; i < docs.size(); i++) { - searcher.doc(iter.nextDoc(), fieldFilter); + docFetcher.doc(iter.nextDoc(), fieldFilter); } } } @@ -404,12 +406,13 @@ public static NamedList getExplanations( Query query, DocList docs, SolrIndexSearcher searcher, IndexSchema schema) throws IOException { + SolrDocumentFetcher docFetcher = searcher.getDocFetcher(); NamedList explainList = new SimpleOrderedMap<>(); DocIterator iterator = docs.iterator(); for (int i = 0; i < docs.size(); i++) { int id = iterator.nextDoc(); - Document doc = searcher.doc(id); + Document doc = docFetcher.doc(id); String strid = schema.printableUniqueKey(doc); explainList.add(strid, searcher.explain(query, id)); diff --git a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java index ffb6f69ee29..f92fbe71e03 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java +++ b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java @@ -56,6 +56,7 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.PointRangeQuery; @@ -5811,8 +5812,9 @@ private void doTestInternals(String field, String[] values) throws IOException { } for (LeafReaderContext leave : ir.leaves()) { LeafReader reader = leave.reader(); + StoredFields storedFields = reader.storedFields(); for (int i = 0; i < reader.numDocs(); i++) { - Document doc = reader.document(i); + Document doc = storedFields.document(i); if (sf.stored()) { assertNotNull("Field " + field + " not found. Doc: " + doc, doc.get(field)); } else { diff --git a/solr/core/src/test/org/apache/solr/search/TestDocSet.java b/solr/core/src/test/org/apache/solr/search/TestDocSet.java index 901a188fedd..a554ce2f409 100644 --- a/solr/core/src/test/org/apache/solr/search/TestDocSet.java +++ b/solr/core/src/test/org/apache/solr/search/TestDocSet.java @@ -303,6 +303,7 @@ public int maxDoc() { } @Override + @Deprecated public Fields getTermVectors(int docID) { return null; } @@ -394,6 +395,7 @@ public void searchNearestVectors( protected void doClose() {} @Override + @Deprecated public void document(int doc, StoredFieldVisitor visitor) {} @Override diff --git a/solr/core/src/test/org/apache/solr/search/TestStressLucene.java b/solr/core/src/test/org/apache/solr/search/TestStressLucene.java index a02b55cb92f..2fc620f8468 100644 --- a/solr/core/src/test/org/apache/solr/search/TestStressLucene.java +++ b/solr/core/src/test/org/apache/solr/search/TestStressLucene.java @@ -347,7 +347,7 @@ public void run() { verbose("ERROR: Couldn't find a doc for id", id, "using reader", r); } assertTrue(docid >= 0); // we should have found the document, or its tombstone - Document doc = r.document(docid); + Document doc = r.storedFields().document(docid); long foundVal = Long.parseLong(doc.get(FIELD)); if (foundVal < Math.abs(val)) { verbose( diff --git a/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java b/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java index 5c37805b9e4..4081fc3b671 100644 --- a/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java +++ b/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java @@ -27,6 +27,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.queries.function.FunctionQuery; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.IndexSearcher; @@ -137,9 +138,10 @@ private void doTestExactScore(String field, boolean inOrder) throws Exception { TopDocs td = s.search(q, 1000); assertEquals("All docs should be matched!", N_DOCS, td.totalHits.value); ScoreDoc sd[] = td.scoreDocs; + StoredFields storedFields = s.getIndexReader().storedFields(); for (int i = 0; i < sd.length; i++) { float score = sd[i].score; - String id = s.getIndexReader().document(sd[i].doc).get(ID_FIELD); + String id = storedFields.document(sd[i].doc).get(ID_FIELD); log("-------- " + i + ". Explain doc " + id); log(s.explain(q, sd[i].doc)); float expectedScore = N_DOCS - i - 1; diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java index 89ae4181f85..ec1525cef7d 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java @@ -38,6 +38,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TermsEnum.SeekStatus; @@ -174,8 +175,9 @@ public void testHugeBinaryValues() throws Exception { TestUtil.checkReader(ar); BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field"); + StoredFields storedFields = ar.storedFields(); for (int docID = 0; docID < docBytes.size(); docID++) { - Document doc = ar.document(docID); + Document doc = storedFields.document(docID); assertEquals(docID, s.nextDoc()); BytesRef bytes = s.binaryValue(); byte[] expected = docBytes.get(Integer.parseInt(doc.get("id"))); diff --git a/solr/modules/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java b/solr/modules/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java index d18e1b7c924..27646190a39 100644 --- a/solr/modules/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java +++ b/solr/modules/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java @@ -55,6 +55,7 @@ import org.apache.solr.search.DocList; import org.apache.solr.search.DocSlice; import org.apache.solr.search.QueryLimits; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.util.plugin.SolrCoreAware; import org.carrot2.clustering.Cluster; @@ -415,13 +416,14 @@ public SolrIndexSearcher getSearcher() { docLanguage = (doc) -> requestParameters.language(); } + SolrDocumentFetcher docFetcher = indexSearcher.getDocFetcher(); List result = new ArrayList<>(); DocIterator it = responseBuilder.getResults().docList.iterator(); while (it.hasNext()) { int docId = it.nextDoc(); Map docFieldValues = new LinkedHashMap<>(); - for (IndexableField indexableField : indexSearcher.doc(docId, fieldsToLoad.keySet())) { + for (IndexableField indexableField : docFetcher.doc(docId, fieldsToLoad.keySet())) { String fieldName = indexableField.name(); Function toString = fieldsToLoad.get(fieldName); if (toString != null) { diff --git a/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/FieldValueFeature.java b/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/FieldValueFeature.java index 583e080954c..02e2fabb5bf 100644 --- a/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/FieldValueFeature.java +++ b/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/FieldValueFeature.java @@ -29,6 +29,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -183,18 +184,21 @@ public FeatureScorer scorer(LeafReaderContext context) throws IOException { public class FieldValueFeatureScorer extends FeatureScorer { private final LeafReaderContext context; + private final StoredFields storedFields; public FieldValueFeatureScorer( - FeatureWeight weight, LeafReaderContext context, DocIdSetIterator itr) { + FeatureWeight weight, LeafReaderContext context, DocIdSetIterator itr) + throws IOException { super(weight, itr); this.context = context; + this.storedFields = (context == null ? null : context.reader().storedFields()); } @Override public float score() throws IOException { try { - final Document document = context.reader().document(itr.docID(), fieldAsSet); + final Document document = storedFields.document(itr.docID(), fieldAsSet); final IndexableField indexableField = document.getField(field); if (indexableField == null) { return getDefaultValue();