From b0d5bcd6b137bc76a59d4a381ff6dc4c9b498b19 Mon Sep 17 00:00:00 2001 From: Keith Grootboom Date: Fri, 2 Dec 2022 08:48:39 +0200 Subject: [PATCH] feat: operators can prefix elasticsearch indexes for multi-tenancy Adds a new configuration setting, ELASTICSEARCH_INDEX_PREFIX. If set, all Elasticsearch indexes and aliases that are generated by the application begin with the prefix. This is to enable multi-tenancy, so that more than one version of this application can run on an ElasticSearch deployment. --- api/search.rb | 4 ++-- app.rb | 14 ++++++++++++-- config/application.yml | 2 ++ lib/task_helpers.rb | 12 ++++++++---- models/comment.rb | 5 ++++- models/comment_thread.rb | 5 ++++- spec/lib/task_helpers_spec.rb | 27 ++++++++++++++++++++++++++- spec/lib/tasks/search_rake_spec.rb | 2 +- 8 files changed, 59 insertions(+), 12 deletions(-) diff --git a/api/search.rb b/api/search.rb index 0e3c8edeeba..4393a510947 100644 --- a/api/search.rb +++ b/api/search.rb @@ -47,7 +47,7 @@ def get_thread_ids(context, group_ids, local_params, search_text) } } - response = Elasticsearch::Model.client.search(index: TaskHelpers::ElasticsearchHelper::INDEX_NAMES, body: body) + response = Elasticsearch::Model.client.search(index: TaskHelpers::ElasticsearchHelper::index_names, body: body) thread_ids = Set.new response['hits']['hits'].each do |hit| @@ -81,7 +81,7 @@ def get_suggested_text(search_text) } } - response = Elasticsearch::Model.client.search(index: TaskHelpers::ElasticsearchHelper::INDEX_NAMES, body: body) + response = Elasticsearch::Model.client.search(index: TaskHelpers::ElasticsearchHelper::index_names, body: body) body_suggestions = response['suggest'].fetch('body_suggestions', []) title_suggestions = response['suggest'].fetch('title_suggestions', []) diff --git a/app.rb b/app.rb index 78dfa1ec518..5196fe9f987 100644 --- a/app.rb +++ b/app.rb @@ -12,6 +12,8 @@ Bundler.setup Bundler.require(*groups) + + logger = Logger.new(STDOUT) logger.level = Logger::WARN begin @@ -66,9 +68,17 @@ def get_logger(progname, threshold=nil) # Example: Elasticsearch::Client.new(tracer: get_logger('elasticsearch.tracer')) # NOTE: You can also add a logger, but it will log some FATAL warning during index creation. # Example: Elasticsearch::Client.new(logger: get_logger('elasticsearch', Logger::WARN)) + +es_config = { log: false, url: CommentService.config[:elasticsearch_server], transport_options: {}} + +if CommentService.config[:elasticsearch_ca_path] + es_config[:transport_options][:ssl] = {ca_file: CommentService.config[:elasticsearch_ca_path] } +end + Elasticsearch::Model.client = Elasticsearch::Client.new( - host: CommentService.config[:elasticsearch_server], - log: false + url: es_config[:url], + log: false, + transport_options: es_config[:transport_options] ) # Setup i18n diff --git a/config/application.yml b/config/application.yml index 4ce627d47b4..acc341deee4 100644 --- a/config/application.yml +++ b/config/application.yml @@ -7,3 +7,5 @@ default_locale: <%= ENV['SERVICE_LANGUAGE'] || 'en-US' %> manual_pagination_batch_size: <%= ENV['MANUAL_PAGINATION_BATCH_SIZE'] || 500 %> thread_response_default_size: <%= ENV['THREAD_RESPONSE_DEFAULT_SIZE'] || 100 %> thread_response_size_limit: <%= ENV['THREAD_RESPONSE_SIZE_LIMIT'] || 200 %> +elasticsearch_index_prefix: <%= ENV['ELASTICSEARCH_INDEX_PREFIX'] || "" %> +elasticsearch_ca_path: <%= ENV['ELASTICSEARCH_CA_PATH'] || "" %> diff --git a/lib/task_helpers.rb b/lib/task_helpers.rb index 9f067959a74..0c1db3e32c2 100644 --- a/lib/task_helpers.rb +++ b/lib/task_helpers.rb @@ -6,8 +6,12 @@ module TaskHelpers module ElasticsearchHelper LOG = Logger.new(STDERR) INDEX_MODELS = [Comment, CommentThread].freeze - INDEX_NAMES = [Comment.index_name, CommentThread.index_name].freeze # local variable which store actual indices for future deletion + + def self.index_names + [Comment.index_name, CommentThread.index_name].freeze + end + @@temporary_index_names = [] def self.temporary_index_names @@ -186,7 +190,7 @@ def self.move_alias(alias_name, index_name, force_delete=false) def self.refresh_indices if temporary_index_names.length > 0 - Elasticsearch::Model.client.indices.refresh(index: INDEX_NAMES) + Elasticsearch::Model.client.indices.refresh(index: self.index_names) else fail "No indices to refresh" end @@ -194,7 +198,7 @@ def self.refresh_indices def self.initialize_indices(force_new_index = false) # When force_new_index is true, fresh indices will be created even if it already exists. - if force_new_index or not exists_aliases(INDEX_NAMES) + if force_new_index or not exists_aliases(self.index_names) index_names = create_indices index_names.each do |index_name| model = get_index_model_rel(index_name) @@ -211,7 +215,7 @@ def self.initialize_indices(force_new_index = false) # Validates that each index includes the proper mappings. # There is no return value, but an exception is raised if the index is invalid. def self.validate_indices - actual_mappings = Elasticsearch::Model.client.indices.get_mapping(index: INDEX_NAMES) + actual_mappings = Elasticsearch::Model.client.indices.get_mapping(index: self.index_names) if actual_mappings.length == 0 fail "Indices are not exist!" diff --git a/models/comment.rb b/models/comment.rb index e39e4030b99..e16aff1f1a1 100644 --- a/models/comment.rb +++ b/models/comment.rb @@ -33,7 +33,10 @@ class Comment < Content index({_type: 1, comment_thread_id: 1, author_id: 1, updated_at: 1}) index({comment_thread_id: 1, author_id: 1, created_at: 1}) - index_name = "comment" + index_name do + prefix = ::CommentService.config[:elasticsearch_index_prefix] + "#{prefix}comments" + end mapping dynamic: 'false' do indexes :body, type: :text, store: true, term_vector: :with_positions_offsets diff --git a/models/comment_thread.rb b/models/comment_thread.rb index 306447d25dc..15f7a4cc3a1 100644 --- a/models/comment_thread.rb +++ b/models/comment_thread.rb @@ -39,7 +39,10 @@ class CommentThread < Content index({ author_id: 1, course_id: 1 }) - index_name = "comment_thread" + index_name do + prefix = ::CommentService.config[:elasticsearch_index_prefix] + "#{prefix}comment_threads" + end mapping dynamic: 'false' do indexes :title, type: :text, boost: 5.0, store: true, term_vector: :with_positions_offsets diff --git a/spec/lib/task_helpers_spec.rb b/spec/lib/task_helpers_spec.rb index e155036f46f..287663d3bd1 100644 --- a/spec/lib/task_helpers_spec.rb +++ b/spec/lib/task_helpers_spec.rb @@ -47,7 +47,7 @@ def assert_alias_points_to_index(alias_name, index_name) create(:comment_thread, body: 'the best test body', course_id: 'test_course_id') TaskHelpers::ElasticsearchHelper.refresh_indices expect(Elasticsearch::Model.client.search( - index: TaskHelpers::ElasticsearchHelper::INDEX_NAMES + index: TaskHelpers::ElasticsearchHelper::index_names )['hits']['total']['value']).to be > 0 end @@ -68,5 +68,30 @@ def assert_alias_points_to_index(alias_name, index_name) end + context("#validate_prefixes") do + subject { TaskHelpers::ElasticsearchHelper} + PREFIX = 'prefix_' + + before(:each) do + CommentService.config[:elasticsearch_index_prefix] = PREFIX + end + + after(:each) do + CommentService.config[:elasticsearch_index_prefix] = "" + end + + it "fails if comment model isn't prefixed" do + expect(Comment.index_name).to start_with(PREFIX) + end + + it "fails if comment thread model isn't prefixed" do + expect(CommentThread.index_name).to start_with(PREFIX) + end + + it "fails if indexes created indexes aren't prefixed" do + expect(TaskHelpers::ElasticsearchHelper.index_names.all? {|v| v.start_with?(PREFIX)} ).to be_truthy + end + end + end end diff --git a/spec/lib/tasks/search_rake_spec.rb b/spec/lib/tasks/search_rake_spec.rb index f656c00ac6c..aedb6c0cfb5 100644 --- a/spec/lib/tasks/search_rake_spec.rb +++ b/spec/lib/tasks/search_rake_spec.rb @@ -30,7 +30,7 @@ describe "search:catchup" do include_context "rake" - let(:indices) { TaskHelpers::ElasticsearchHelper::INDEX_NAMES } + let(:indices) { TaskHelpers::ElasticsearchHelper::index_names } let(:comments_index_name) { Comment.index_name } let(:comment_threads_index_name) { CommentThread.index_name }