From 55f241affec669dd9f0f1b45e994277feb5c2f08 Mon Sep 17 00:00:00 2001 From: Mike Alhayek Date: Tue, 5 Dec 2023 15:08:05 -0800 Subject: [PATCH] Add a way to provide a custom Elasticsearch query Fix #14842 --- .../Drivers/ElasticSettingsDisplayDriver.cs | 83 +++++++++++++++++-- .../Services/ElasticSearchService.cs | 51 ++++++++++-- .../ViewModels/ElasticSettingsViewModel.cs | 13 ++- .../Views/ElasticSettings.Edit.cshtml | 74 ++++++++++++----- .../OrchardCore.Abstractions/JsonHelpers.cs | 33 ++++++++ .../Models/ElasticSettings.cs | 7 +- .../Services/ElasticIndexManager.cs | 2 +- .../Services/ElasticQueryService.cs | 8 +- 8 files changed, 224 insertions(+), 47 deletions(-) create mode 100644 src/OrchardCore/OrchardCore.Abstractions/JsonHelpers.cs diff --git a/src/OrchardCore.Modules/OrchardCore.Search.Elasticsearch/Drivers/ElasticSettingsDisplayDriver.cs b/src/OrchardCore.Modules/OrchardCore.Search.Elasticsearch/Drivers/ElasticSettingsDisplayDriver.cs index 869bea6bdf0..11c82f19527 100644 --- a/src/OrchardCore.Modules/OrchardCore.Search.Elasticsearch/Drivers/ElasticSettingsDisplayDriver.cs +++ b/src/OrchardCore.Modules/OrchardCore.Search.Elasticsearch/Drivers/ElasticSettingsDisplayDriver.cs @@ -1,13 +1,20 @@ using System; +using System.IO; using System.Linq; +using System.Text; +using System.Text.Json; using System.Threading.Tasks; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Localization; +using Nest; using OrchardCore.DisplayManagement.Entities; using OrchardCore.DisplayManagement.Handlers; using OrchardCore.DisplayManagement.Views; +using OrchardCore.Mvc.ModelBinding; using OrchardCore.Search.Elasticsearch.Core.Models; using OrchardCore.Search.Elasticsearch.Core.Services; +using OrchardCore.Search.Elasticsearch.Services; using OrchardCore.Search.Elasticsearch.ViewModels; using OrchardCore.Settings; @@ -16,19 +23,33 @@ namespace OrchardCore.Search.Elasticsearch.Drivers public class ElasticSettingsDisplayDriver : SectionDisplayDriver { public const string GroupId = "elasticsearch"; + + private static readonly char[] _separator = [',', ' ']; + private static readonly JsonSerializerOptions _jsonSerializerOptions = new() + { + WriteIndented = true, + + }; private readonly ElasticIndexSettingsService _elasticIndexSettingsService; private readonly IHttpContextAccessor _httpContextAccessor; private readonly IAuthorizationService _authorizationService; + private readonly IElasticClient _elasticClient; + protected readonly IStringLocalizer S; public ElasticSettingsDisplayDriver( ElasticIndexSettingsService elasticIndexSettingsService, IHttpContextAccessor httpContextAccessor, - IAuthorizationService authorizationService + IAuthorizationService authorizationService, + IElasticClient elasticClient, + IStringLocalizer stringLocalizer + ) { _elasticIndexSettingsService = elasticIndexSettingsService; _httpContextAccessor = httpContextAccessor; _authorizationService = authorizationService; + _elasticClient = elasticClient; + S = stringLocalizer; } public override async Task EditAsync(ElasticSettings settings, BuildEditorContext context) @@ -41,12 +62,30 @@ public override async Task EditAsync(ElasticSettings settings, B } return Initialize("ElasticSettings_Edit", async model => + { + model.SearchIndex = settings.SearchIndex; + model.SearchFields = string.Join(", ", settings.DefaultSearchFields ?? []); + model.SearchIndexes = (await _elasticIndexSettingsService.GetSettingsAsync()).Select(x => x.IndexName); + model.DefaultQuery = settings.DefaultQuery; + +#pragma warning disable CS0618 // Type or member is obsolete + if (settings.SearchType == null && settings.AllowElasticQueryStringQueryInSearch) { - model.SearchIndex = settings.SearchIndex; - model.SearchFields = string.Join(", ", settings.DefaultSearchFields ?? Array.Empty()); - model.SearchIndexes = (await _elasticIndexSettingsService.GetSettingsAsync()).Select(x => x.IndexName); - model.AllowElasticQueryStringQueryInSearch = settings.AllowElasticQueryStringQueryInSearch; - }).Location("Content:2").OnGroup(GroupId); + model.SearchType = "query_string"; + } + else + { + model.SearchType = settings.SearchType; + } +#pragma warning restore CS0618 // Type or member is obsolete + + model.SearchTypes = [ + new(S["Multi-match query"], string.Empty), + new(S["String query"], ElasticsearchService.QueryStringSearchType), + new(S["Raw query"], ElasticsearchService.RawSearchType), + ]; + }).Location("Content:2") + .OnGroup(GroupId); } public override async Task UpdateAsync(ElasticSettings section, BuildEditorContext context) @@ -64,9 +103,37 @@ public override async Task UpdateAsync(ElasticSettings section, await context.Updater.TryUpdateModelAsync(model, Prefix); + section.DefaultQuery = null; section.SearchIndex = model.SearchIndex; - section.DefaultSearchFields = model.SearchFields?.Split(new[] { ',', ' ' }, StringSplitOptions.RemoveEmptyEntries); - section.AllowElasticQueryStringQueryInSearch = model.AllowElasticQueryStringQueryInSearch; + section.DefaultSearchFields = model.SearchFields?.Split(_separator, StringSplitOptions.RemoveEmptyEntries); + section.SearchType = model.SearchType ?? string.Empty; + + if (model.SearchType == ElasticsearchService.RawSearchType) + { + if (string.IsNullOrWhiteSpace(model.DefaultQuery)) + { + context.Updater.ModelState.AddModelError(Prefix, nameof(model.DefaultQuery), S["Please provide the default query."]); + } + else if (!JsonHelpers.TryParse(model.DefaultQuery, out var document)) + { + context.Updater.ModelState.AddModelError(Prefix, nameof(model.DefaultQuery), S["The provided query is not formatted correctly."]); + } + else + { + section.DefaultQuery = JsonSerializer.Serialize(document, _jsonSerializerOptions); + + try + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(model.DefaultQuery)); + + var searchRequest = await _elasticClient.RequestResponseSerializer.DeserializeAsync(stream); + } + catch + { + context.Updater.ModelState.AddModelError(Prefix, nameof(model.DefaultQuery), S["Invalid query provided."]); + } + } + } } return await EditAsync(section, context); diff --git a/src/OrchardCore.Modules/OrchardCore.Search.Elasticsearch/Services/ElasticSearchService.cs b/src/OrchardCore.Modules/OrchardCore.Search.Elasticsearch/Services/ElasticSearchService.cs index 32a03be3032..cb3b6ac0e5d 100644 --- a/src/OrchardCore.Modules/OrchardCore.Search.Elasticsearch/Services/ElasticSearchService.cs +++ b/src/OrchardCore.Modules/OrchardCore.Search.Elasticsearch/Services/ElasticSearchService.cs @@ -1,8 +1,13 @@ using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Text.Encodings.Web; using System.Threading.Tasks; +using Fluid.Values; using Microsoft.Extensions.Logging; using Nest; -using OrchardCore.Entities; +using OrchardCore.Liquid; using OrchardCore.Search.Abstractions; using OrchardCore.Search.Elasticsearch.Core.Models; using OrchardCore.Search.Elasticsearch.Core.Services; @@ -12,10 +17,16 @@ namespace OrchardCore.Search.Elasticsearch.Services; public class ElasticsearchService : ISearchService { + public const string RawSearchType = "raw"; + public const string QueryStringSearchType = "query_string"; + private readonly ISiteService _siteService; private readonly ElasticIndexManager _elasticIndexManager; private readonly ElasticIndexSettingsService _elasticIndexSettingsService; private readonly IElasticSearchQueryService _elasticsearchQueryService; + private readonly IElasticClient _elasticClient; + private readonly JavaScriptEncoder _javaScriptEncoder; + private readonly ILiquidTemplateManager _liquidTemplateManager; private readonly ILogger _logger; public ElasticsearchService( @@ -23,6 +34,9 @@ public ElasticsearchService( ElasticIndexManager elasticIndexManager, ElasticIndexSettingsService elasticIndexSettingsService, IElasticSearchQueryService elasticsearchQueryService, + IElasticClient elasticClient, + JavaScriptEncoder javaScriptEncoder, + ILiquidTemplateManager liquidTemplateManager, ILogger logger ) { @@ -30,6 +44,9 @@ ILogger logger _elasticIndexManager = elasticIndexManager; _elasticIndexSettingsService = elasticIndexSettingsService; _elasticsearchQueryService = elasticsearchQueryService; + _elasticClient = elasticClient; + _javaScriptEncoder = javaScriptEncoder; + _liquidTemplateManager = liquidTemplateManager; _logger = logger; } @@ -65,18 +82,27 @@ public async Task SearchAsync(string indexName, string term, int s { QueryContainer query = null; - if (searchSettings.AllowElasticQueryStringQueryInSearch) + if (searchSettings.SearchType == RawSearchType) { - query = new QueryStringQuery + var tokenizedContent = await _liquidTemplateManager.RenderStringAsync(searchSettings.DefaultQuery, _javaScriptEncoder, + new Dictionary() + { + ["term"] = new StringValue(term) + }); + + try { - Fields = searchSettings.DefaultSearchFields, - Analyzer = await _elasticIndexSettingsService.GetQueryAnalyzerAsync(index), - Query = term - }; + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(tokenizedContent)); + + var searchRequest = await _elasticClient.RequestResponseSerializer.DeserializeAsync(stream); + + query = searchRequest.Query; + } + catch { } } - else + else if (searchSettings.SearchType == "query_string") { - query = new MultiMatchQuery + query = new QueryStringQuery { Fields = searchSettings.DefaultSearchFields, Analyzer = await _elasticIndexSettingsService.GetQueryAnalyzerAsync(index), @@ -84,6 +110,13 @@ public async Task SearchAsync(string indexName, string term, int s }; } + query ??= new MultiMatchQuery + { + Fields = searchSettings.DefaultSearchFields, + Analyzer = await _elasticIndexSettingsService.GetQueryAnalyzerAsync(index), + Query = term + }; + result.ContentItemIds = await _elasticsearchQueryService.ExecuteQueryAsync(index, query, null, start, pageSize); result.Success = true; } diff --git a/src/OrchardCore.Modules/OrchardCore.Search.Elasticsearch/ViewModels/ElasticSettingsViewModel.cs b/src/OrchardCore.Modules/OrchardCore.Search.Elasticsearch/ViewModels/ElasticSettingsViewModel.cs index 5e6383c8951..c06bc3f4ff1 100644 --- a/src/OrchardCore.Modules/OrchardCore.Search.Elasticsearch/ViewModels/ElasticSettingsViewModel.cs +++ b/src/OrchardCore.Modules/OrchardCore.Search.Elasticsearch/ViewModels/ElasticSettingsViewModel.cs @@ -1,13 +1,24 @@ using System.Collections.Generic; +using Microsoft.AspNetCore.Mvc.ModelBinding; +using Microsoft.AspNetCore.Mvc.Rendering; namespace OrchardCore.Search.Elasticsearch.ViewModels { public class ElasticSettingsViewModel { public string Analyzer { get; set; } + public string SearchIndex { get; set; } + public IEnumerable SearchIndexes { get; set; } + public string SearchFields { get; set; } - public bool AllowElasticQueryStringQueryInSearch { get; set; } + + public string DefaultQuery { get; set; } + + public string SearchType { get; set; } + + [BindNever] + public IEnumerable SearchTypes { get; set; } } } diff --git a/src/OrchardCore.Modules/OrchardCore.Search.Elasticsearch/Views/ElasticSettings.Edit.cshtml b/src/OrchardCore.Modules/OrchardCore.Search.Elasticsearch/Views/ElasticSettings.Edit.cshtml index 7dc6c6c7974..9642b3a95a9 100644 --- a/src/OrchardCore.Modules/OrchardCore.Search.Elasticsearch/Views/ElasticSettings.Edit.cshtml +++ b/src/OrchardCore.Modules/OrchardCore.Search.Elasticsearch/Views/ElasticSettings.Edit.cshtml @@ -1,35 +1,63 @@ +@using OrchardCore.Search.Elasticsearch.Services + @model ElasticSettingsViewModel -@if (Model.SearchIndexes.Any()) -{ -
- - - - @T["The default index to use for the search page."] -
-} -else +@if (!Model.SearchIndexes.Any()) {
@T["You need to create at least an index to set as the Search index."]
+ + return; } -
+
+ + + + @T["The default index to use for the search page."] +
+ +
+ + + +
+ +
+ + + + @T["Create a custom Elasticsearch query to be utilized for each search request. Liquid is supported, so use {0} template as a substitute for the user-provided search term.", "{{ term }}"] +
+ +
@T["A comma separated list of fields to use for search pages. The default value is Content.ContentItem.FullText."]
-
-
- - - @T["Whether search queries should be allowed to use Elasticsearch \"query string query\" syntax."] @T["See documentation"] -
-
+ diff --git a/src/OrchardCore/OrchardCore.Abstractions/JsonHelpers.cs b/src/OrchardCore/OrchardCore.Abstractions/JsonHelpers.cs new file mode 100644 index 00000000000..371b1778ad3 --- /dev/null +++ b/src/OrchardCore/OrchardCore.Abstractions/JsonHelpers.cs @@ -0,0 +1,33 @@ +using System.Text.Json; + +namespace OrchardCore; + +public class JsonHelpers +{ + public static bool IsValid(string json, JsonDocumentOptions options = default) + { + try + { + JsonDocument.Parse(json, options); + + return true; + } + catch { } + + return false; + } + + public static bool TryParse(string json, out JsonDocument document, JsonDocumentOptions options = default) + { + try + { + document = JsonDocument.Parse(json, options); + + return true; + } + catch { } + + document = null; + return false; + } +} diff --git a/src/OrchardCore/OrchardCore.Search.Elasticsearch.Core/Models/ElasticSettings.cs b/src/OrchardCore/OrchardCore.Search.Elasticsearch.Core/Models/ElasticSettings.cs index acd5500dd1f..b942715f858 100644 --- a/src/OrchardCore/OrchardCore.Search.Elasticsearch.Core/Models/ElasticSettings.cs +++ b/src/OrchardCore/OrchardCore.Search.Elasticsearch.Core/Models/ElasticSettings.cs @@ -5,15 +5,20 @@ namespace OrchardCore.Search.Elasticsearch.Core.Models { public class ElasticSettings { - public static readonly string[] FullTextField = new string[] { IndexingConstants.FullTextKey }; + public static readonly string[] FullTextField = [IndexingConstants.FullTextKey]; [Obsolete("This property will be removed in future releases.")] public const string StandardAnalyzer = "standardanalyzer"; public string SearchIndex { get; set; } + public string DefaultQuery { get; set; } + public string[] DefaultSearchFields { get; set; } = FullTextField; + [Obsolete("Instead use SearchType property.")] public bool AllowElasticQueryStringQueryInSearch { get; set; } = false; + + public string SearchType { get; set; } } } diff --git a/src/OrchardCore/OrchardCore.Search.Elasticsearch.Core/Services/ElasticIndexManager.cs b/src/OrchardCore/OrchardCore.Search.Elasticsearch.Core/Services/ElasticIndexManager.cs index d14732d9295..881717f898d 100644 --- a/src/OrchardCore/OrchardCore.Search.Elasticsearch.Core/Services/ElasticIndexManager.cs +++ b/src/OrchardCore/OrchardCore.Search.Elasticsearch.Core/Services/ElasticIndexManager.cs @@ -102,7 +102,7 @@ public async Task CreateIndexAsync(ElasticIndexSettings elasticIndexSettin var indexSettingsDescriptor = new IndexSettingsDescriptor(); // The name "standardanalyzer" is a legacy used prior OC 1.6 release. It can be removed in future releases. - var analyzerName = elasticIndexSettings.AnalyzerName == "standardanalyzer" ? "standard" : elasticIndexSettings.AnalyzerName; + var analyzerName = (elasticIndexSettings.AnalyzerName == "standardanalyzer" ? null : elasticIndexSettings.AnalyzerName) ?? "standard"; if (_elasticsearchOptions.Analyzers.TryGetValue(analyzerName, out var analyzerProperties)) { diff --git a/src/OrchardCore/OrchardCore.Search.Elasticsearch.Core/Services/ElasticQueryService.cs b/src/OrchardCore/OrchardCore.Search.Elasticsearch.Core/Services/ElasticQueryService.cs index ea3817561a7..12243eee9e9 100644 --- a/src/OrchardCore/OrchardCore.Search.Elasticsearch.Core/Services/ElasticQueryService.cs +++ b/src/OrchardCore/OrchardCore.Search.Elasticsearch.Core/Services/ElasticQueryService.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Text; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -38,7 +37,8 @@ public async Task SearchAsync(string indexName, string query) try { - var deserializedSearchRequest = _elasticClient.RequestResponseSerializer.Deserialize(new MemoryStream(Encoding.UTF8.GetBytes(query))); + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(query)); + var deserializedSearchRequest = _elasticClient.RequestResponseSerializer.Deserialize(stream); var searchRequest = new SearchRequest(_indexPrefix + indexName) { @@ -47,7 +47,7 @@ public async Task SearchAsync(string indexName, string query) Size = deserializedSearchRequest.Size, Fields = deserializedSearchRequest.Fields, Sort = deserializedSearchRequest.Sort, - Source = deserializedSearchRequest.Source + Source = deserializedSearchRequest.Source, }; var searchResponse = await _elasticClient.SearchAsync>(searchRequest); @@ -71,7 +71,7 @@ public async Task SearchAsync(string indexName, string query) if (searchResponse.IsValid) { elasticTopDocs.Count = searchResponse.Total; - elasticTopDocs.TopDocs = searchResponse.Documents.ToList(); + elasticTopDocs.TopDocs = [.. searchResponse.Documents]; elasticTopDocs.Fields = hits; } else