SOLR-16677: Update Solr to use new Lucene 9.5 storedFields() API (#1557)

Co-authored-by: Christine Poerschke <[email protected]> Co-authored-by: Michael Gibney <[email protected]>
apache · Jun 26, 2024 · 502faf2 · 502faf2
1 parent 7b00dda
commit 502faf2
Show file tree

Hide file tree

Showing 29 changed files with 233 additions and 115 deletions.
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
@@ -146,6 +146,10 @@ Optimizations
 
 * SOLR-17330: When not set, loadOnStartup defaults to true, which is the default choice for a core. (Pierre Salagnac via Eric Pugh)
 
+* SOLR-16677: Update Solr to use new Lucene 9.5 storedFields() API. This removes the use of ThreadLocal for
+  stored field state, reducing heap usage especially for high-core-count, high-field-count, high-thread-count
+  cases (Vinayak Hegde, Christine Poerschke, Kevin Risden, David Smiley, Michael Gibney)
+
 Bug Fixes
 ---------------------
 * SOLR-12813: subqueries should respect basic auth. (Rudy Seitz via Eric Pugh)

diff --git a/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java b/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java
@@ -30,6 +30,7 @@
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.search.DocIterator;
 import org.apache.solr.search.DocList;
+import org.apache.solr.search.SolrDocumentFetcher;
 import org.apache.solr.search.SolrIndexSearcher;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -80,13 +81,19 @@ public void close() {}
           NamedList<?> values = rsp.getValues();
           for (int i = 0; i < values.size(); i++) {
             Object o = values.getVal(i);
+            SolrDocumentFetcher docFetcher = null;
             if (o instanceof ResultContext) {
-              o = ((ResultContext) o).getDocList();
+              ResultContext ctx = (ResultContext) o;
+              o = ctx.getDocList();
+              docFetcher = ctx.getDocFetcher();
             }
             if (o instanceof DocList) {
               DocList docs = (DocList) o;
+              if (docFetcher == null) {
+                docFetcher = newSearcher.getDocFetcher();
+              }
               for (DocIterator iter = docs.iterator(); iter.hasNext(); ) {
-                newSearcher.doc(iter.nextDoc());
+                docFetcher.doc(iter.nextDoc());
               }
             }
           }

diff --git a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
@@ -140,7 +140,7 @@ public void handleRequestBody(final SolrQueryRequest req, SolrQueryResponse rsp)
 
         long version = 0;
         if (docs.totalHits.value > 0) {
-          Document doc = req.getSearcher().doc(docs.scoreDocs[0].doc);
+          Document doc = req.getSearcher().getDocFetcher().doc(docs.scoreDocs[0].doc);
           Number n = doc.getField("version").numericValue();
           version = n.longValue();
         }
@@ -214,7 +214,7 @@ public void handleRequestBody(final SolrQueryRequest req, SolrQueryResponse rsp)
 
                   @Override
                   public void write(OutputStream os) throws IOException {
-                    Document doc = req.getSearcher().doc(docs.scoreDocs[0].doc);
+                    Document doc = req.getSearcher().getDocFetcher().doc(docs.scoreDocs[0].doc);
                     IndexableField sf = doc.getField("blob");
                     FieldType fieldType = req.getSchema().getField("blob").getType();
                     ByteBuffer buf = (ByteBuffer) fieldType.toObject(sf);

diff --git a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
@@ -26,9 +26,11 @@
 import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
+import net.jcip.annotations.NotThreadSafe;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.ExitableDirectoryReader;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.mlt.MoreLikeThis;
 import org.apache.lucene.search.BooleanClause;
@@ -282,17 +284,20 @@ public static class InterestingTerm {
   }
 
   /** Helper class for MoreLikeThis that can be called from other request handlers */
+  @NotThreadSafe
   public static class MoreLikeThisHelper {
     final SolrIndexSearcher searcher;
     final MoreLikeThis mlt;
     final IndexReader reader;
+    final StoredFields storedFields;
     final SchemaField uniqueKeyField;
     final boolean needDocSet;
     Map<String, Float> boostFields;
 
     public MoreLikeThisHelper(SolrParams params, SolrIndexSearcher searcher) throws IOException {
       this.searcher = searcher;
       this.reader = searcher.getIndexReader();
+      this.storedFields = this.reader.storedFields();
       this.uniqueKeyField = searcher.getSchema().getUniqueKeyField();
       this.needDocSet = params.getBool(FacetParams.FACET, false);
 
@@ -394,7 +399,7 @@ private BooleanQuery getBoostedQuery(Query mltquery) {
 
     public DocListAndSet getMoreLikeThis(
         int id, int start, int rows, List<Query> filters, int flags) throws IOException {
-      Document doc = reader.document(id);
+      Document doc = this.storedFields.document(id);
       final Query boostedQuery = getBoostedMLTQuery(id);
 
       // exclude current document from results

diff --git a/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java b/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java
@@ -45,6 +45,8 @@
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StandardDirectoryReader;
 import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.StoredFields;
+import org.apache.lucene.index.TermVectors;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.DocIdSetIterator;
@@ -384,11 +386,12 @@ private void estimateTermVectors(Map<String, Object> result) throws IOException
     for (LeafReaderContext leafReaderContext : reader.leaves()) {
       LeafReader leafReader = leafReaderContext.reader();
       Bits liveDocs = leafReader.getLiveDocs();
+      TermVectors leafTermVectors = leafReader.termVectors();
       for (int docId = 0; docId < leafReader.maxDoc(); docId += samplingStep) {
         if (liveDocs != null && !liveDocs.get(docId)) {
           continue;
         }
-        Fields termVectors = leafReader.getTermVectors(docId);
+        Fields termVectors = leafTermVectors.get(docId);
         if (termVectors == null) {
           continue;
         }
@@ -608,11 +611,12 @@ private void estimateStoredFields(Map<String, Object> result) throws IOException
           mergeInstance.close();
         }
       } else {
+        StoredFields storedFields = leafReader.storedFields();
         for (int docId = 0; docId < leafReader.maxDoc(); docId += samplingStep) {
           if (liveDocs != null && !liveDocs.get(docId)) {
             continue;
           }
-          leafReader.document(docId, visitor);
+          storedFields.document(docId, visitor);
         }
       }
     }

diff --git a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
@@ -52,7 +52,9 @@
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.MultiTerms;
 import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermVectors;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.DocIdSetIterator;
@@ -161,7 +163,7 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw
       }
       Document doc = null;
       try {
-        doc = reader.document(docId);
+        doc = reader.storedFields().document(docId);
       } catch (Exception ex) {
       }
       if (doc == null) {
@@ -316,6 +318,7 @@ private static SimpleOrderedMap<Object> getDocumentFieldsInfo(
       Document doc, int docId, IndexReader reader, IndexSchema schema) throws IOException {
     final CharsRefBuilder spare = new CharsRefBuilder();
     SimpleOrderedMap<Object> finfo = new SimpleOrderedMap<>();
+    TermVectors termVectors = null;
     for (Object o : doc.getFields()) {
       Field field = (Field) o;
       SimpleOrderedMap<Object> f = new SimpleOrderedMap<>();
@@ -353,7 +356,8 @@ private static SimpleOrderedMap<Object> getDocumentFieldsInfo(
       // If we have a term vector, return that
       if (field.fieldType().storeTermVectors()) {
         try {
-          Terms v = reader.getTermVector(docId, field.name());
+          if (termVectors == null) termVectors = reader.termVectors();
+          Terms v = termVectors.get(docId, field.name());
           if (v != null) {
             SimpleOrderedMap<Integer> tfv = new SimpleOrderedMap<>();
             final TermsEnum termsEnum = v.iterator();
@@ -462,6 +466,7 @@ private static Document getFirstLiveDoc(Terms terms, LeafReader reader) throws I
     PostingsEnum postingsEnum = null;
     TermsEnum termsEnum = terms.iterator();
     BytesRef text;
+    StoredFields storedFields = reader.storedFields();
     // Deal with the chance that the first bunch of terms are in deleted documents. Is there a
     // better way?
     for (int idx = 0; idx < 1000 && postingsEnum == null; ++idx) {
@@ -476,7 +481,7 @@ private static Document getFirstLiveDoc(Terms terms, LeafReader reader) throws I
         if (liveDocs != null && liveDocs.get(postingsEnum.docID())) {
           continue;
         }
-        return reader.document(postingsEnum.docID());
+        return storedFields.document(postingsEnum.docID());
       }
     }
     return null;

diff --git a/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java b/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java
@@ -49,6 +49,7 @@
 import org.apache.solr.search.DocListAndSet;
 import org.apache.solr.search.QueryLimits;
 import org.apache.solr.search.ReturnFields;
+import org.apache.solr.search.SolrDocumentFetcher;
 import org.apache.solr.search.SolrIndexSearcher;
 import org.apache.solr.search.SolrReturnFields;
 import org.slf4j.Logger;
@@ -103,6 +104,7 @@ public void process(ResponseBuilder rb) throws IOException {
               new MoreLikeThisHandler.MoreLikeThisHelper(params, searcher);
           NamedList<NamedList<?>> mltQueryByDocKey = new NamedList<>();
           QueryLimits queryLimits = QueryLimits.getCurrentLimits();
+          SolrDocumentFetcher docFetcher = rb.req.getSearcher().getDocFetcher();
           for (DocIterator results = rb.getResults().docList.iterator(); results.hasNext(); ) {
             int docId = results.nextDoc();
             final List<MoreLikeThisHandler.InterestingTerm> interestingTerms =
@@ -114,7 +116,7 @@ public void process(ResponseBuilder rb) throws IOException {
               break;
             }
             final String uniqueKey = rb.req.getSchema().getUniqueKeyField().getName();
-            final Document document = rb.req.getSearcher().doc(docId);
+            final Document document = docFetcher.doc(docId);
             final String uniqueVal = rb.req.getSchema().printableUniqueKey(document);
             final NamedList<String> mltQ =
                 mltViaQueryParams(rb.req.getSchema(), interestingTerms, uniqueKey, uniqueVal);
@@ -423,12 +425,13 @@ NamedList<DocList> getMoreLikeThese(
 
     QueryLimits queryLimits = QueryLimits.getCurrentLimits();
 
+    SolrDocumentFetcher docFetcher = searcher.getDocFetcher();
     while (iterator.hasNext()) {
       int id = iterator.nextDoc();
       int rows = p.getInt(MoreLikeThisParams.DOC_COUNT, 5);
 
       DocListAndSet similarDocuments = mltHelper.getMoreLikeThis(id, 0, rows, null, flags);
-      String name = schema.printableUniqueKey(searcher.doc(id));
+      String name = schema.printableUniqueKey(docFetcher.doc(id));
       mltResponse.add(name, similarDocuments.docList);
 
       if (dbg != null) {
@@ -440,7 +443,7 @@ NamedList<DocList> getMoreLikeThese(
         DocIterator similarDocumentsIterator = similarDocuments.docList.iterator();
         while (similarDocumentsIterator.hasNext()) {
           int mltid = similarDocumentsIterator.nextDoc();
-          String key = schema.printableUniqueKey(searcher.doc(mltid));
+          String key = schema.printableUniqueKey(docFetcher.doc(mltid));
           explains.add(key, searcher.explain(mltHelper.getRealMLTQuery(), mltid));
         }
         docDbg.add("explain", explains);

diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
@@ -347,10 +347,10 @@ public void process(ResponseBuilder rb) throws IOException {
 
         if (docid < 0) continue;
 
+        SolrDocumentFetcher docFetcher = searcherInfo.getSearcher().getDocFetcher();
         Document luceneDocument =
-            searcherInfo.getSearcher().doc(docid, rsp.getReturnFields().getLuceneFieldNames());
+            docFetcher.doc(docid, rsp.getReturnFields().getLuceneFieldNames());
         SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema());
-        SolrDocumentFetcher docFetcher = searcherInfo.getSearcher().getDocFetcher();
         if (reuseDvIters == null) {
           reuseDvIters = new DocValuesIteratorCache(searcherInfo.getSearcher());
         }

diff --git a/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java b/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java
@@ -24,6 +24,7 @@
 import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.search.DocIterator;
 import org.apache.solr.search.DocList;
+import org.apache.solr.search.SolrDocumentFetcher;
 import org.apache.solr.search.SolrIndexSearcher;
 
 /**
@@ -90,9 +91,10 @@ protected void processIds(
     StringBuilder sb = new StringBuilder();
 
     Set<String> fields = Collections.singleton(schema.getUniqueKeyField().getName());
+    SolrDocumentFetcher docFetcher = searcher.getDocFetcher();
     for (DocIterator iter = dl.iterator(); iter.hasNext(); ) {
 
-      sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields))).append(',');
+      sb.append(schema.printableUniqueKey(docFetcher.doc(iter.nextDoc(), fields))).append(',');
     }
     if (sb.length() > 0) {
       rb.rsp.addToLog("responseLog", sb.substring(0, sb.length() - 1));
@@ -105,8 +107,9 @@ protected void processScores(
 
     StringBuilder sb = new StringBuilder();
     Set<String> fields = Collections.singleton(schema.getUniqueKeyField().getName());
+    SolrDocumentFetcher docFetcher = searcher.getDocFetcher();
     for (DocIterator iter = dl.iterator(); iter.hasNext(); ) {
-      sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields)))
+      sb.append(schema.printableUniqueKey(docFetcher.doc(iter.nextDoc(), fields)))
           .append(':')
           .append(iter.score())
           .append(',');

diff --git a/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java b/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java
@@ -34,6 +34,7 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.PostingsEnum;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermVectors;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.BytesRef;
@@ -251,6 +252,7 @@ public void process(ResponseBuilder rb) throws IOException {
     SolrIndexSearcher searcher = rb.req.getSearcher();
 
     IndexReader reader = searcher.getIndexReader();
+    TermVectors termVectors = reader.termVectors();
     // the TVMapper is a TermVectorMapper which can be used to optimize loading of Term Vectors
 
     // Only load the id field to get the uniqueKey of that
@@ -277,15 +279,15 @@ public void process(ResponseBuilder rb) throws IOException {
       if (null != fields) {
         for (Map.Entry<String, FieldOptions> entry : fieldOptions.entrySet()) {
           final String field = entry.getKey();
-          final Terms vector = reader.getTermVector(docId, field);
+          final Terms vector = termVectors.get(docId, field);
           if (vector != null) {
             TermsEnum termsEnum = vector.iterator();
             mapOneVector(docNL, entry.getValue(), reader, docId, termsEnum, field);
           }
         }
       } else {
         // extract all fields
-        final Fields vectors = reader.getTermVectors(docId);
+        final Fields vectors = termVectors.get(docId);
         // There can be no documents with vectors
         if (vectors != null) {
           for (String field : vectors) {