Recherche via Elasticsearch

requirements.txt

@@ -2,6 +2,8 @@
 pysolr==3.4.0
 pygments==2.1.3
 python-social-auth==0.2.19
+elasticsearch>=2.0.0,<3.0.0
+elasticsearch-dsl>=5.0.0,<6.0.0
 
 # Explicit dependencies (references in code)
 Django==1.8.16

zds/forum/migrations/0011_auto_20161224_1310.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('forum', '0010_auto_20161112_1823'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='post',
+            name='es_already_indexed',
+            field=models.BooleanField(default=False, db_index=True, verbose_name=b'D\xc3\xa9j\xc3\xa0 index\xc3\xa9 par ES'),
+        ),
+        migrations.AddField(
+            model_name='post',
+            name='es_flagged',
+            field=models.BooleanField(default=True, db_index=True, verbose_name=b'Doit \xc3\xaatre (r\xc3\xa9)index\xc3\xa9 par ES'),
+        ),
+        migrations.AddField(
+            model_name='topic',
+            name='es_already_indexed',
+            field=models.BooleanField(default=False, db_index=True, verbose_name=b'D\xc3\xa9j\xc3\xa0 index\xc3\xa9 par ES'),
+        ),
+        migrations.AddField(
+            model_name='topic',
+            name='es_flagged',
+            field=models.BooleanField(default=True, db_index=True, verbose_name=b'Doit \xc3\xaatre (r\xc3\xa9)index\xc3\xa9 par ES'),
+        ),
+    ]

zds/forum/models.py

@@ -10,9 +10,12 @@
 from django.core.urlresolvers import reverse
 from django.db import models
 
+from elasticsearch_dsl.field import Text, Keyword
+
 from zds.forum.managers import TopicManager, ForumManager, PostManager, TopicReadManager
 from zds.notification import signals
 from zds.settings import ZDS_APP
+from zds.search2.models import AbstractESDjangoIndexable
 from zds.utils import get_current_user
 from zds.utils import slugify
 from zds.utils.models import Comment, Tag
@@ -157,7 +160,7 @@ def can_read(self, user):
 
 
 @python_2_unicode_compatible
-class Topic(models.Model):
+class Topic(AbstractESDjangoIndexable):
     """
     A Topic is a thread of posts.
     A topic has several states, witch are all independent:
@@ -382,9 +385,39 @@ def old_post_warning(self):
 
         return False
 
+    @classmethod
+    def get_es_mapping(cls):
+        es_mapping = super(Topic, cls).get_es_mapping()
+
+        es_mapping.field('title', Text())
+        es_mapping.field('subtitle', Text())
+        es_mapping.field('get_absolute_url', Text(index='not_analyzed'))
+        es_mapping.field('tags', Keyword())
+
+        return es_mapping
+
+    @classmethod
+    def get_es_django_indexable(cls, force_reindexing=False):
+        """Overridden to remove hidden forums (and prefetch tags)
+        """
+
+        query = super(Topic, cls).get_es_django_indexable(force_reindexing)
+        return query.prefetch_related('tags').filter(forum__group__isnull=True)
+
+    def get_es_document_source(self, excluded_fields=None):
+        """Overridden to handle the case of tags (M2M field)
+        """
+
+        excluded_fields = excluded_fields or []
+        excluded_fields.extend(['tags'])
+
+        data = super(Topic, self).get_es_document_source(excluded_fields=excluded_fields)
+        data['tags'] = [tag.title for tag in self.tags.all()]
+        return data
+
 
 @python_2_unicode_compatible
-class Post(Comment):
+class Post(Comment, AbstractESDjangoIndexable):
     """
     A forum post written by an user.
     A post can be marked as useful: topic's author (or admin) can declare any topic as "useful", and this post is
@@ -413,6 +446,23 @@ def get_absolute_url(self):
     def get_notification_title(self):
         return self.topic.title
 
+    @classmethod
+    def get_es_mapping(cls):
+        m = super(Post, cls).get_es_mapping()
+
+        m.field('text', Text())
+        m.field('get_absolute_url', Text(index='not_analyzed'))
+
+        return m
+
+    @classmethod
+    def get_es_django_indexable(cls, force_reindexing=False):
+        """Overridden to remove invisible post
+        """
+
+        q = super(Post, cls).get_es_django_indexable(force_reindexing)
+        return q.filter(is_visible=True)
+
 
 @python_2_unicode_compatible
 class TopicRead(models.Model):

zds/search2/__init__.py

@@ -0,0 +1,25 @@
+# coding: utf-8
+from django.conf import settings
+from elasticsearch_dsl.connections import connections
+
+DEFAULT_ES_CONNECTIONS = {
+    'default': {
+        'hosts': ['localhost:9200'],
+    }
+}
+
+INDEX_NAME = getattr(settings, 'ES_INDEX_NAME', 'elastic')
+CONNECTIONS = getattr(settings, 'ES_CONNECTIONS', DEFAULT_ES_CONNECTIONS)
+
+
+def setup_es_connections():
+    """Create connection(s) to Elasticsearch from parameters defined in the settings.
+
+    CONNECTIONS is a dict, where the keys are connection aliases and the values are parameters to the
+    ``elasticsearch_dsl.connections.connection.create_connection()`` function (which are directly passed to an
+    Elasticsearch object, see http://elasticsearch-py.readthedocs.io/en/master/api.html#elasticsearch for the options).
+
+    """
+
+    for alias, params in CONNECTIONS.items():
+        connections.create_connection(alias, **params)

zds/search2/management/commands/es_manager.py

@@ -0,0 +1,48 @@
+# coding: utf-8
+
+from django.core.management.base import BaseCommand, CommandError
+
+from zds.search2 import INDEX_NAME, setup_es_connections
+from zds.search2.models import ESIndexManager, get_django_indexable_objects
+from zds.tutorialv2.models.models_database import FakeChapter
+
+
+class Command(BaseCommand):
+    help = 'Manage data from/to ES'
+
+    indexer = None
+    models = get_django_indexable_objects()
+
+    def __init__(self, *args, **kwargs):
+        super(Command, self).__init__(*args, **kwargs)
+
+        self.models.insert(0, FakeChapter)  # FakeChapter needs to be first
+
+    def add_arguments(self, parser):
+        parser.add_argument('action', type=str)
+
+    def handle(self, *args, **options):
+        setup_es_connections()
+        self.indexer = ESIndexManager(INDEX_NAME)
+
+        if options['action'] == 'setup':
+            self.setup_es()
+        elif options['action'] == 'index-all':
+            self.index_documents(force_reindexing=True)
+        elif options['action'] == 'index-flagged':
+            self.index_documents(force_reindexing=False)
+        else:
+            raise CommandError('unknown action {}'.format(options['action']))
+
+    def setup_es(self):
+
+        self.indexer.reset_es_index()
+        self.indexer.setup_es_mappings(self.models)
+
+    def index_documents(self, force_reindexing=False):
+
+        if force_reindexing:
+            self.setup_es()  # remove all previous data
+
+        for model in self.models:
+            self.indexer.es_bulk_indexing_of_model(model, force_reindexing=force_reindexing)