From dbb2f051530d62517c153ee6059b84f388b84b28 Mon Sep 17 00:00:00 2001 From: Mike Alhayek Date: Mon, 26 Aug 2024 12:17:42 -0700 Subject: [PATCH] Add a base document-index to allow indexing non-content items in AzureAI search (#16611) --- .../Services/LuceneIndexManager.cs | 18 ++--- .../DocumentIndex.cs | 74 ++----------------- .../DocumentIndexBase.cs | 72 ++++++++++++++++++ .../Models/AzureAISearchIndexMap.cs | 2 +- .../Models/SearchIndexDefinition.cs | 2 +- .../AzureAISearchIndexDocumentManager.cs | 20 ++--- .../Services/AzureAISearchIndexManager.cs | 2 +- .../Services/AzureAISearchIndexingService.cs | 2 +- 8 files changed, 102 insertions(+), 90 deletions(-) create mode 100644 src/OrchardCore/OrchardCore.Indexing.Abstractions/DocumentIndexBase.cs diff --git a/src/OrchardCore.Modules/OrchardCore.Search.Lucene/Services/LuceneIndexManager.cs b/src/OrchardCore.Modules/OrchardCore.Search.Lucene/Services/LuceneIndexManager.cs index 8580091bd4b..d83375effa3 100644 --- a/src/OrchardCore.Modules/OrchardCore.Search.Lucene/Services/LuceneIndexManager.cs +++ b/src/OrchardCore.Modules/OrchardCore.Search.Lucene/Services/LuceneIndexManager.cs @@ -206,7 +206,7 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe switch (entry.Type) { - case DocumentIndex.Types.Boolean: + case DocumentIndexBase.Types.Boolean: // Store "true"/"false" for boolean. doc.Add(new StringField(entry.Name, Convert.ToString(entry.Value).ToLowerInvariant(), store)); @@ -216,7 +216,7 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe } break; - case DocumentIndex.Types.DateTime: + case DocumentIndexBase.Types.DateTime: if (entry.Value != null) { if (entry.Value is DateTimeOffset) @@ -246,7 +246,7 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe } break; - case DocumentIndex.Types.Integer: + case DocumentIndexBase.Types.Integer: if (entry.Value != null && long.TryParse(entry.Value.ToString(), out var value)) { doc.Add(new Int64Field(entry.Name, value, store)); @@ -263,7 +263,7 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe break; - case DocumentIndex.Types.Number: + case DocumentIndexBase.Types.Number: if (entry.Value != null) { doc.Add(new DoubleField(entry.Name, Convert.ToDouble(entry.Value), store)); @@ -279,7 +279,7 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe } break; - case DocumentIndex.Types.Text: + case DocumentIndexBase.Types.Text: if (entry.Value != null && !string.IsNullOrEmpty(Convert.ToString(entry.Value))) { var stringValue = Convert.ToString(entry.Value); @@ -319,10 +319,10 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe } break; - case DocumentIndex.Types.GeoPoint: + case DocumentIndexBase.Types.GeoPoint: var strategy = new RecursivePrefixTreeStrategy(_grid, entry.Name); - if (entry.Value != null && entry.Value is DocumentIndex.GeoPoint point) + if (entry.Value != null && entry.Value is DocumentIndexBase.GeoPoint point) { var geoPoint = _ctx.MakePoint((double)point.Longitude, (double)point.Latitude); foreach (var field in strategy.CreateIndexableFields(geoPoint)) @@ -422,7 +422,7 @@ private IndexReaderPool.IndexReaderLease GetReader(string indexName) } /// - /// Releases all readers and writers. This can be used after some time of innactivity to free resources. + /// Releases all readers and writers. This can be used after some time of inactivity to free resources. /// public void FreeReaderWriter() { @@ -456,7 +456,7 @@ public void FreeReaderWriter() } /// - /// Releases all readers and writers. This can be used after some time of innactivity to free resources. + /// Releases all readers and writers. This can be used after some time of inactivity to free resources. /// public void FreeReaderWriter(string indexName) { diff --git a/src/OrchardCore/OrchardCore.Indexing.Abstractions/DocumentIndex.cs b/src/OrchardCore/OrchardCore.Indexing.Abstractions/DocumentIndex.cs index 31b88e756c0..e73c965073a 100644 --- a/src/OrchardCore/OrchardCore.Indexing.Abstractions/DocumentIndex.cs +++ b/src/OrchardCore/OrchardCore.Indexing.Abstractions/DocumentIndex.cs @@ -1,76 +1,14 @@ -using Microsoft.AspNetCore.Html; - namespace OrchardCore.Indexing; -public class DocumentIndex(string contentItemId, string contentItemVersionId) +public class DocumentIndex : DocumentIndexBase { - public List Entries { get; } = []; - - public void Set(string name, string value, DocumentIndexOptions options) - { - Entries.Add(new DocumentIndexEntry(name, value, Types.Text, options)); - } - - public void Set(string name, IHtmlContent value, DocumentIndexOptions options) - { - Entries.Add(new DocumentIndexEntry(name, value, Types.Text, options)); - } - - public void Set(string name, DateTimeOffset? value, DocumentIndexOptions options) - { - Entries.Add(new DocumentIndexEntry(name, value, Types.DateTime, options)); - } - - public void Set(string name, int? value, DocumentIndexOptions options) - { - Entries.Add(new DocumentIndexEntry(name, value, Types.Integer, options)); - } - - public void Set(string name, bool? value, DocumentIndexOptions options) - { - Entries.Add(new DocumentIndexEntry(name, value, Types.Boolean, options)); - } + public string ContentItemId { get; } - public void Set(string name, double? value, DocumentIndexOptions options) - { - Entries.Add(new DocumentIndexEntry(name, value, Types.Number, options)); - } - - public void Set(string name, decimal? value, DocumentIndexOptions options) - { - Entries.Add(new DocumentIndexEntry(name, value, Types.Number, options)); - } - - public void Set(string name, GeoPoint value, DocumentIndexOptions options) - { - Entries.Add(new DocumentIndexEntry(name, value, Types.GeoPoint, options)); - } - - public string ContentItemId { get; } = contentItemId; - - public string ContentItemVersionId { get; } = contentItemVersionId; - - public enum Types - { - Integer, - Text, - DateTime, - Boolean, - Number, - GeoPoint - } - - public class GeoPoint - { - public decimal Longitude; - public decimal Latitude; - } + public string ContentItemVersionId { get; } - public class DocumentIndexEntry(string name, object value, Types type, DocumentIndexOptions options) + public DocumentIndex(string contentItemId, string contentItemVersionId) { - public string Name { get; } = name; - public object Value { get; } = value; - public Types Type { get; } = type; - public DocumentIndexOptions Options { get; } = options; + ContentItemId = contentItemId; + ContentItemVersionId = contentItemVersionId; } } diff --git a/src/OrchardCore/OrchardCore.Indexing.Abstractions/DocumentIndexBase.cs b/src/OrchardCore/OrchardCore.Indexing.Abstractions/DocumentIndexBase.cs new file mode 100644 index 00000000000..3e885882b9d --- /dev/null +++ b/src/OrchardCore/OrchardCore.Indexing.Abstractions/DocumentIndexBase.cs @@ -0,0 +1,72 @@ +using Microsoft.AspNetCore.Html; + +namespace OrchardCore.Indexing; + +public class DocumentIndexBase +{ + public List Entries { get; } = []; + + public void Set(string name, string value, DocumentIndexOptions options) + { + Entries.Add(new DocumentIndexEntry(name, value, Types.Text, options)); + } + + public void Set(string name, IHtmlContent value, DocumentIndexOptions options) + { + Entries.Add(new DocumentIndexEntry(name, value, Types.Text, options)); + } + + public void Set(string name, DateTimeOffset? value, DocumentIndexOptions options) + { + Entries.Add(new DocumentIndexEntry(name, value, Types.DateTime, options)); + } + + public void Set(string name, int? value, DocumentIndexOptions options) + { + Entries.Add(new DocumentIndexEntry(name, value, Types.Integer, options)); + } + + public void Set(string name, bool? value, DocumentIndexOptions options) + { + Entries.Add(new DocumentIndexEntry(name, value, Types.Boolean, options)); + } + + public void Set(string name, double? value, DocumentIndexOptions options) + { + Entries.Add(new DocumentIndexEntry(name, value, Types.Number, options)); + } + + public void Set(string name, decimal? value, DocumentIndexOptions options) + { + Entries.Add(new DocumentIndexEntry(name, value, Types.Number, options)); + } + + public void Set(string name, GeoPoint value, DocumentIndexOptions options) + { + Entries.Add(new DocumentIndexEntry(name, value, Types.GeoPoint, options)); + } + + public enum Types + { + Integer, + Text, + DateTime, + Boolean, + Number, + GeoPoint, + } + + public class GeoPoint + { + public decimal Longitude; + public decimal Latitude; + } + + public class DocumentIndexEntry(string name, object value, Types type, DocumentIndexOptions options) + { + public string Name { get; } = name; + public object Value { get; } = value; + public Types Type { get; } = type; + public DocumentIndexOptions Options { get; } = options; + } +} diff --git a/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Models/AzureAISearchIndexMap.cs b/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Models/AzureAISearchIndexMap.cs index 1ffab766398..2bb23643918 100644 --- a/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Models/AzureAISearchIndexMap.cs +++ b/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Models/AzureAISearchIndexMap.cs @@ -1,5 +1,5 @@ using OrchardCore.Indexing; -using static OrchardCore.Indexing.DocumentIndex; +using static OrchardCore.Indexing.DocumentIndexBase; namespace OrchardCore.Search.AzureAI.Models; diff --git a/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Models/SearchIndexDefinition.cs b/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Models/SearchIndexDefinition.cs index 2bd1dc8c8bd..8adf96ac752 100644 --- a/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Models/SearchIndexDefinition.cs +++ b/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Models/SearchIndexDefinition.cs @@ -1,4 +1,4 @@ -using static OrchardCore.Indexing.DocumentIndex; +using static OrchardCore.Indexing.DocumentIndexBase; namespace OrchardCore.Search.AzureAI.Models; diff --git a/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Services/AzureAISearchIndexDocumentManager.cs b/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Services/AzureAISearchIndexDocumentManager.cs index 9b40b868dd6..14568462756 100644 --- a/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Services/AzureAISearchIndexDocumentManager.cs +++ b/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Services/AzureAISearchIndexDocumentManager.cs @@ -6,7 +6,7 @@ using OrchardCore.Indexing; using OrchardCore.Modules; using OrchardCore.Search.AzureAI.Models; -using static OrchardCore.Indexing.DocumentIndex; +using static OrchardCore.Indexing.DocumentIndexBase; namespace OrchardCore.Search.AzureAI.Services; @@ -130,7 +130,7 @@ public async Task DeleteAllDocumentsAsync(string indexName) await DeleteDocumentsAsync(indexName, contentItemIds); } - public async Task MergeOrUploadDocumentsAsync(string indexName, IList indexDocuments, AzureAISearchIndexSettings indexSettings) + public async Task MergeOrUploadDocumentsAsync(string indexName, IList indexDocuments, AzureAISearchIndexSettings indexSettings) { ArgumentException.ThrowIfNullOrEmpty(indexName); ArgumentNullException.ThrowIfNull(indexDocuments); @@ -244,7 +244,7 @@ private async Task AddIndexMappingAsync(List indexMapping indexMappings.Add(indexMap); } - private static IEnumerable CreateSearchDocuments(IEnumerable indexDocuments, Dictionary> mappings) + private static IEnumerable CreateSearchDocuments(IEnumerable indexDocuments, Dictionary> mappings) { foreach (var indexDocument in indexDocuments) { @@ -252,13 +252,15 @@ private static IEnumerable CreateSearchDocuments(IEnumerable> mappingDictionary) + private static SearchDocument CreateSearchDocument(DocumentIndexBase documentIndex, Dictionary> mappingDictionary) { - var doc = new SearchDocument() + var doc = new SearchDocument(); + + if (documentIndex is DocumentIndex index) { - { IndexingConstants.ContentItemIdKey, documentIndex.ContentItemId }, - { IndexingConstants.ContentItemVersionIdKey, documentIndex.ContentItemVersionId }, - }; + doc.Add(IndexingConstants.ContentItemIdKey, index.ContentItemId); + doc.Add(IndexingConstants.ContentItemVersionIdKey, index.ContentItemVersionId); + } foreach (var entry in documentIndex.Entries) { @@ -276,7 +278,7 @@ private static SearchDocument CreateSearchDocument(DocumentIndex documentIndex, switch (entry.Type) { - case Types.Boolean: + case DocumentIndexBase.Types.Boolean: if (entry.Value is bool boolValue) { doc.TryAdd(map.AzureFieldKey, boolValue); diff --git a/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Services/AzureAISearchIndexManager.cs b/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Services/AzureAISearchIndexManager.cs index b9505a40180..1fa9e266c22 100644 --- a/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Services/AzureAISearchIndexManager.cs +++ b/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Services/AzureAISearchIndexManager.cs @@ -6,7 +6,7 @@ using OrchardCore.Environment.Shell; using OrchardCore.Modules; using OrchardCore.Search.AzureAI.Models; -using static OrchardCore.Indexing.DocumentIndex; +using static OrchardCore.Indexing.DocumentIndexBase; namespace OrchardCore.Search.AzureAI.Services; diff --git a/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Services/AzureAISearchIndexingService.cs b/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Services/AzureAISearchIndexingService.cs index 1a757250cc3..374989cf149 100644 --- a/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Services/AzureAISearchIndexingService.cs +++ b/src/OrchardCore/OrchardCore.Search.AzureAI.Core/Services/AzureAISearchIndexingService.cs @@ -93,7 +93,7 @@ public async Task ProcessContentItemsAsync(params string[] indexNames) Dictionary allLatest = null; // Group all DocumentIndex by index to batch update them. - var updatedDocumentsByIndex = indexSettings.ToDictionary(x => x.IndexName, b => new List()); + var updatedDocumentsByIndex = indexSettings.ToDictionary(x => x.IndexName, b => new List()); var settingsByIndex = indexSettings.ToDictionary(x => x.IndexName);