Skip to content

Commit

Permalink
Add a base document-index to allow indexing non-content items in Azur…
Browse files Browse the repository at this point in the history
…eAI search (#16611)
  • Loading branch information
MikeAlhayek authored Aug 26, 2024
1 parent 4395dbb commit dbb2f05
Show file tree
Hide file tree
Showing 8 changed files with 102 additions and 90 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe

switch (entry.Type)
{
case DocumentIndex.Types.Boolean:
case DocumentIndexBase.Types.Boolean:
// Store "true"/"false" for boolean.
doc.Add(new StringField(entry.Name, Convert.ToString(entry.Value).ToLowerInvariant(), store));

Expand All @@ -216,7 +216,7 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe
}
break;

case DocumentIndex.Types.DateTime:
case DocumentIndexBase.Types.DateTime:
if (entry.Value != null)
{
if (entry.Value is DateTimeOffset)
Expand Down Expand Up @@ -246,7 +246,7 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe
}
break;

case DocumentIndex.Types.Integer:
case DocumentIndexBase.Types.Integer:
if (entry.Value != null && long.TryParse(entry.Value.ToString(), out var value))
{
doc.Add(new Int64Field(entry.Name, value, store));
Expand All @@ -263,7 +263,7 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe

break;

case DocumentIndex.Types.Number:
case DocumentIndexBase.Types.Number:
if (entry.Value != null)
{
doc.Add(new DoubleField(entry.Name, Convert.ToDouble(entry.Value), store));
Expand All @@ -279,7 +279,7 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe
}
break;

case DocumentIndex.Types.Text:
case DocumentIndexBase.Types.Text:
if (entry.Value != null && !string.IsNullOrEmpty(Convert.ToString(entry.Value)))
{
var stringValue = Convert.ToString(entry.Value);
Expand Down Expand Up @@ -319,10 +319,10 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe
}
break;

case DocumentIndex.Types.GeoPoint:
case DocumentIndexBase.Types.GeoPoint:
var strategy = new RecursivePrefixTreeStrategy(_grid, entry.Name);

if (entry.Value != null && entry.Value is DocumentIndex.GeoPoint point)
if (entry.Value != null && entry.Value is DocumentIndexBase.GeoPoint point)
{
var geoPoint = _ctx.MakePoint((double)point.Longitude, (double)point.Latitude);
foreach (var field in strategy.CreateIndexableFields(geoPoint))
Expand Down Expand Up @@ -422,7 +422,7 @@ private IndexReaderPool.IndexReaderLease GetReader(string indexName)
}

/// <summary>
/// Releases all readers and writers. This can be used after some time of innactivity to free resources.
/// Releases all readers and writers. This can be used after some time of inactivity to free resources.
/// </summary>
public void FreeReaderWriter()
{
Expand Down Expand Up @@ -456,7 +456,7 @@ public void FreeReaderWriter()
}

/// <summary>
/// Releases all readers and writers. This can be used after some time of innactivity to free resources.
/// Releases all readers and writers. This can be used after some time of inactivity to free resources.
/// </summary>
public void FreeReaderWriter(string indexName)
{
Expand Down
74 changes: 6 additions & 68 deletions src/OrchardCore/OrchardCore.Indexing.Abstractions/DocumentIndex.cs
Original file line number Diff line number Diff line change
@@ -1,76 +1,14 @@
using Microsoft.AspNetCore.Html;

namespace OrchardCore.Indexing;

public class DocumentIndex(string contentItemId, string contentItemVersionId)
public class DocumentIndex : DocumentIndexBase
{
public List<DocumentIndexEntry> Entries { get; } = [];

public void Set(string name, string value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Text, options));
}

public void Set(string name, IHtmlContent value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Text, options));
}

public void Set(string name, DateTimeOffset? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.DateTime, options));
}

public void Set(string name, int? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Integer, options));
}

public void Set(string name, bool? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Boolean, options));
}
public string ContentItemId { get; }

public void Set(string name, double? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Number, options));
}

public void Set(string name, decimal? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Number, options));
}

public void Set(string name, GeoPoint value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.GeoPoint, options));
}

public string ContentItemId { get; } = contentItemId;

public string ContentItemVersionId { get; } = contentItemVersionId;

public enum Types
{
Integer,
Text,
DateTime,
Boolean,
Number,
GeoPoint
}

public class GeoPoint
{
public decimal Longitude;
public decimal Latitude;
}
public string ContentItemVersionId { get; }

public class DocumentIndexEntry(string name, object value, Types type, DocumentIndexOptions options)
public DocumentIndex(string contentItemId, string contentItemVersionId)
{
public string Name { get; } = name;
public object Value { get; } = value;
public Types Type { get; } = type;
public DocumentIndexOptions Options { get; } = options;
ContentItemId = contentItemId;
ContentItemVersionId = contentItemVersionId;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
using Microsoft.AspNetCore.Html;

namespace OrchardCore.Indexing;

public class DocumentIndexBase
{
public List<DocumentIndexEntry> Entries { get; } = [];

public void Set(string name, string value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Text, options));
}

public void Set(string name, IHtmlContent value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Text, options));
}

public void Set(string name, DateTimeOffset? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.DateTime, options));
}

public void Set(string name, int? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Integer, options));
}

public void Set(string name, bool? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Boolean, options));
}

public void Set(string name, double? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Number, options));
}

public void Set(string name, decimal? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Number, options));
}

public void Set(string name, GeoPoint value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.GeoPoint, options));
}

public enum Types
{
Integer,
Text,
DateTime,
Boolean,
Number,
GeoPoint,
}

public class GeoPoint
{
public decimal Longitude;
public decimal Latitude;
}

public class DocumentIndexEntry(string name, object value, Types type, DocumentIndexOptions options)
{
public string Name { get; } = name;
public object Value { get; } = value;
public Types Type { get; } = type;
public DocumentIndexOptions Options { get; } = options;
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
using OrchardCore.Indexing;
using static OrchardCore.Indexing.DocumentIndex;
using static OrchardCore.Indexing.DocumentIndexBase;

namespace OrchardCore.Search.AzureAI.Models;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using static OrchardCore.Indexing.DocumentIndex;
using static OrchardCore.Indexing.DocumentIndexBase;

namespace OrchardCore.Search.AzureAI.Models;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
using OrchardCore.Indexing;
using OrchardCore.Modules;
using OrchardCore.Search.AzureAI.Models;
using static OrchardCore.Indexing.DocumentIndex;
using static OrchardCore.Indexing.DocumentIndexBase;

namespace OrchardCore.Search.AzureAI.Services;

Expand Down Expand Up @@ -130,7 +130,7 @@ public async Task DeleteAllDocumentsAsync(string indexName)
await DeleteDocumentsAsync(indexName, contentItemIds);
}

public async Task<bool> MergeOrUploadDocumentsAsync(string indexName, IList<DocumentIndex> indexDocuments, AzureAISearchIndexSettings indexSettings)
public async Task<bool> MergeOrUploadDocumentsAsync(string indexName, IList<DocumentIndexBase> indexDocuments, AzureAISearchIndexSettings indexSettings)
{
ArgumentException.ThrowIfNullOrEmpty(indexName);
ArgumentNullException.ThrowIfNull(indexDocuments);
Expand Down Expand Up @@ -244,21 +244,23 @@ private async Task AddIndexMappingAsync(List<AzureAISearchIndexMap> indexMapping
indexMappings.Add(indexMap);
}

private static IEnumerable<SearchDocument> CreateSearchDocuments(IEnumerable<DocumentIndex> indexDocuments, Dictionary<string, IEnumerable<AzureAISearchIndexMap>> mappings)
private static IEnumerable<SearchDocument> CreateSearchDocuments(IEnumerable<DocumentIndexBase> indexDocuments, Dictionary<string, IEnumerable<AzureAISearchIndexMap>> mappings)
{
foreach (var indexDocument in indexDocuments)
{
yield return CreateSearchDocument(indexDocument, mappings);
}
}

private static SearchDocument CreateSearchDocument(DocumentIndex documentIndex, Dictionary<string, IEnumerable<AzureAISearchIndexMap>> mappingDictionary)
private static SearchDocument CreateSearchDocument(DocumentIndexBase documentIndex, Dictionary<string, IEnumerable<AzureAISearchIndexMap>> mappingDictionary)
{
var doc = new SearchDocument()
var doc = new SearchDocument();

if (documentIndex is DocumentIndex index)
{
{ IndexingConstants.ContentItemIdKey, documentIndex.ContentItemId },
{ IndexingConstants.ContentItemVersionIdKey, documentIndex.ContentItemVersionId },
};
doc.Add(IndexingConstants.ContentItemIdKey, index.ContentItemId);
doc.Add(IndexingConstants.ContentItemVersionIdKey, index.ContentItemVersionId);
}

foreach (var entry in documentIndex.Entries)
{
Expand All @@ -276,7 +278,7 @@ private static SearchDocument CreateSearchDocument(DocumentIndex documentIndex,

switch (entry.Type)
{
case Types.Boolean:
case DocumentIndexBase.Types.Boolean:
if (entry.Value is bool boolValue)
{
doc.TryAdd(map.AzureFieldKey, boolValue);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
using OrchardCore.Environment.Shell;
using OrchardCore.Modules;
using OrchardCore.Search.AzureAI.Models;
using static OrchardCore.Indexing.DocumentIndex;
using static OrchardCore.Indexing.DocumentIndexBase;

namespace OrchardCore.Search.AzureAI.Services;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ public async Task ProcessContentItemsAsync(params string[] indexNames)
Dictionary<string, ContentItem> allLatest = null;

// Group all DocumentIndex by index to batch update them.
var updatedDocumentsByIndex = indexSettings.ToDictionary(x => x.IndexName, b => new List<DocumentIndex>());
var updatedDocumentsByIndex = indexSettings.ToDictionary(x => x.IndexName, b => new List<DocumentIndexBase>());

var settingsByIndex = indexSettings.ToDictionary(x => x.IndexName);

Expand Down

0 comments on commit dbb2f05

Please sign in to comment.