Merge pull request #404 from open-craft/keith/prefix-elasticsearch-in…

…dexes

feat: operators can prefix elasticsearch indexes for multi-tenancy

api/search.rb

@@ -47,7 +47,7 @@ def get_thread_ids(context, group_ids, local_params, search_text)
     }
   }
 
-  response = Elasticsearch::Model.client.search(index: TaskHelpers::ElasticsearchHelper::INDEX_NAMES, body: body)
+  response = Elasticsearch::Model.client.search(index: TaskHelpers::ElasticsearchHelper::index_names, body: body)
 
   thread_ids = Set.new
   response['hits']['hits'].each do |hit|
@@ -81,7 +81,7 @@ def get_suggested_text(search_text)
     }
   }
 
-  response = Elasticsearch::Model.client.search(index: TaskHelpers::ElasticsearchHelper::INDEX_NAMES, body: body)
+  response = Elasticsearch::Model.client.search(index: TaskHelpers::ElasticsearchHelper::index_names, body: body)
   body_suggestions = response['suggest'].fetch('body_suggestions', [])
   title_suggestions = response['suggest'].fetch('title_suggestions', [])
 

app.rb

@@ -67,8 +67,9 @@ def get_logger(progname, threshold=nil)
 # NOTE: You can also add a logger, but it will log some FATAL warning during index creation.
 # Example: Elasticsearch::Client.new(logger: get_logger('elasticsearch', Logger::WARN))
 Elasticsearch::Model.client = Elasticsearch::Client.new(
-    host: CommentService.config[:elasticsearch_server],
-    log: false
+    url: CommentService.config[:elasticsearch_server],
+    log: false,
+    transport_options: CommentService.config[:elasticsearch_transport_options],
 )
 
 # Setup i18n

config/application.yml

@@ -7,3 +7,11 @@ default_locale: <%= ENV['SERVICE_LANGUAGE'] || 'en-US' %>
 manual_pagination_batch_size: <%= ENV['MANUAL_PAGINATION_BATCH_SIZE'] || 500 %>
 thread_response_default_size: <%= ENV['THREAD_RESPONSE_DEFAULT_SIZE'] || 100 %>
 thread_response_size_limit: <%= ENV['THREAD_RESPONSE_SIZE_LIMIT'] || 200 %>
+elasticsearch_index_prefix: <%= ENV['ELASTICSEARCH_INDEX_PREFIX'] || "" %>
+<% if ENV['ELASTICSEARCH_CA_PATH'] %>
+elasticsearch_transport_options:
+  ssl:
+    ca_file: <%= ENV['ELASTICSEARCH_CA_PATH'] %>
+<% else %>
+elasticsearch_transport_options: {}
+<% end %>

lib/task_helpers.rb

@@ -6,8 +6,12 @@ module TaskHelpers
   module ElasticsearchHelper
     LOG = Logger.new(STDERR)
     INDEX_MODELS = [Comment, CommentThread].freeze
-    INDEX_NAMES = [Comment.index_name, CommentThread.index_name].freeze
     # local variable which store actual indices for future deletion
+
+    def self.index_names
+      [Comment.index_name, CommentThread.index_name].freeze
+    end
+
     @@temporary_index_names = []
 
     def self.temporary_index_names
@@ -186,15 +190,15 @@ def self.move_alias(alias_name, index_name, force_delete=false)
 
     def self.refresh_indices
       if temporary_index_names.length > 0
-        Elasticsearch::Model.client.indices.refresh(index: INDEX_NAMES)
+        Elasticsearch::Model.client.indices.refresh(index: self.index_names)
       else
         fail "No indices to refresh"
       end
     end
 
     def self.initialize_indices(force_new_index = false)
       # When force_new_index is true, fresh indices will be created even if it already exists.
-      if force_new_index or not exists_aliases(INDEX_NAMES)
+      if force_new_index or not exists_aliases(self.index_names)
         index_names = create_indices
         index_names.each do |index_name|
           model = get_index_model_rel(index_name)
@@ -211,7 +215,7 @@ def self.initialize_indices(force_new_index = false)
     # Validates that each index includes the proper mappings.
     # There is no return value, but an exception is raised if the index is invalid.
     def self.validate_indices
-      actual_mappings = Elasticsearch::Model.client.indices.get_mapping(index: INDEX_NAMES)
+      actual_mappings = Elasticsearch::Model.client.indices.get_mapping(index: self.index_names)
 
       if actual_mappings.length == 0
         fail "Indices are not exist!"

models/comment.rb

@@ -33,7 +33,10 @@ class Comment < Content
   index({_type: 1, comment_thread_id: 1, author_id: 1, updated_at: 1})
   index({comment_thread_id: 1, author_id: 1, created_at: 1})
 
-  index_name = "comment"
+  index_name do
+    prefix = ::CommentService.config[:elasticsearch_index_prefix]
+    "#{prefix}comments"
+  end
 
   mapping dynamic: 'false' do
     indexes :body, type: :text, store: true, term_vector: :with_positions_offsets

models/comment_thread.rb

@@ -39,7 +39,10 @@ class CommentThread < Content
 
   index({ author_id: 1, course_id: 1 })
 
-  index_name = "comment_thread"
+  index_name do
+    prefix = ::CommentService.config[:elasticsearch_index_prefix]
+    "#{prefix}comment_threads"
+  end
 
   mapping dynamic: 'false' do
     indexes :title, type: :text, boost: 5.0, store: true, term_vector: :with_positions_offsets

spec/lib/task_helpers_spec.rb

@@ -47,7 +47,7 @@ def assert_alias_points_to_index(alias_name, index_name)
         create(:comment_thread, body: 'the best test body', course_id: 'test_course_id')
         TaskHelpers::ElasticsearchHelper.refresh_indices
         expect(Elasticsearch::Model.client.search(
-            index: TaskHelpers::ElasticsearchHelper::INDEX_NAMES
+            index: TaskHelpers::ElasticsearchHelper::index_names
         )['hits']['total']['value']).to be > 0
       end
 
@@ -68,5 +68,30 @@ def assert_alias_points_to_index(alias_name, index_name)
 
     end
 
+    context("#validate_prefixes") do
+      subject { TaskHelpers::ElasticsearchHelper}
+      PREFIX = 'prefix_'
+
+      before(:each) do
+        CommentService.config[:elasticsearch_index_prefix] = PREFIX
+      end
+
+      after(:each) do
+        CommentService.config[:elasticsearch_index_prefix] = ""
+      end
+
+      it "fails if comment model isn't prefixed" do
+        expect(Comment.index_name).to start_with(PREFIX)
+      end
+
+      it "fails if comment thread model isn't prefixed" do
+        expect(CommentThread.index_name).to start_with(PREFIX)
+      end
+
+      it "fails if indexes created indexes aren't prefixed" do
+        expect(TaskHelpers::ElasticsearchHelper.index_names.all? {|v| v.start_with?(PREFIX)} ).to be_truthy
+      end
+    end
+
   end
 end

spec/lib/tasks/search_rake_spec.rb

@@ -30,7 +30,7 @@
 
 describe "search:catchup" do
   include_context "rake"
-  let(:indices) { TaskHelpers::ElasticsearchHelper::INDEX_NAMES }
+  let(:indices) { TaskHelpers::ElasticsearchHelper::index_names }
   let(:comments_index_name) { Comment.index_name }
   let(:comment_threads_index_name) { CommentThread.index_name }