search pass three

realms/__init__.py

@@ -11,14 +11,14 @@ import httplib
 import traceback
 import click
 from flask import Flask, request, render_template, url_for, redirect, g
-from flask.ext.elastic import Elastic
 from flask.ext.cache import Cache
 from flask.ext.login import LoginManager, current_user
-from flask.ext.sqlalchemy import SQLAlchemy, declarative_base, Model, _QueryProperty
+from flask.ext.sqlalchemy import SQLAlchemy
 from flask.ext.assets import Environment, Bundle
 from werkzeug.routing import BaseConverter
 from werkzeug.exceptions import HTTPException
 
+from .modules.search.models import Search
 from .lib.util import to_canonical, remove_ext, mkdir_safe, gravatar_url, to_dict
 from .lib.hook import HookModelMeta
 from .lib.util import is_su, in_virtualenv
@@ -161,7 +161,7 @@ def create_app(config=None):
     db.init_app(app)
     cache.init_app(app)
     assets.init_app(app)
-    elastic.init_app(app)
+    search.init_app(app)
 
     for status_code in httplib.responses:
         if status_code >= 400:
@@ -199,7 +199,7 @@ login_manager = LoginManager()
 db = SQLAlchemy()
 cache = Cache()
 assets = Assets()
-elastic = Elastic()
+search = Search()
 
 assets.register('main.js',
                 'vendor/jquery/dist/jquery.js',

realms/config/__init__.py

@@ -95,6 +95,9 @@ CACHE_REDIS_DB = '0'
 #CACHE_TYPE = 'memcached'
 CACHE_MEMCACHED_SERVERS = ['127.0.0.1:11211']
 
+SEARCH_TYPE = 'simple'  # simple is not good for large wikis
+
+# SEARCH_TYPE = 'elasticsearch'
 ELASTICSEARCH_URL = 'http://127.0.0.1:9200'
 
 # Get ReCaptcha Keys for your domain here:

realms/lib/util.py

@@ -97,6 +97,25 @@ def to_canonical(s):
     s = s.lower()
     return s
 
+def cname_to_filename(cname):
+    """ Convert canonical name to filename
+
+    :param cname: Canonical name
+    :return: str -- Filename
+
+    """
+    return cname + ".md"
+
+
+def filename_to_cname(filename):
+    """Convert filename to canonical name.
+
+    .. note::
+
+    It's assumed filename is already canonical format
+
+    """
+    return os.path.splitext(filename)[0]
 
 def gravatar_url(email):
     return "//www.gravatar.com/avatar/" + hashlib.md5(email).hexdigest()

realms/modules/search/commands.py

@@ -0,0 +1,36 @@
+import click
+from realms import create_app, search
+from realms.modules.wiki.models import Wiki
+from realms.lib.util import filename_to_cname
+
+
+@click.group(short_help="Search Module")
+def cli():
+    pass
+
+
+@cli.command()
+def rebuild_index():
+    """ Rebuild search index
+    """
+    app = create_app()
+
+    if app.config.get('SEARCH_TYPE') == 'simple':
+        click.echo("Search type is simple, try using elasticsearch.")
+        return
+
+    with app.app_context():
+        # Wiki
+        search.delete_index('wiki')
+        wiki = Wiki(app.config['WIKI_PATH'])
+        for entry in wiki.get_index():
+            page = wiki.get_page(entry['name'])
+            name = filename_to_cname(page['name'])
+            # TODO add email?
+            body = dict(name=name,
+                        content=page['data'],
+                        message=page['info']['message'],
+                        username=page['info']['author'],
+                        updated_on=entry['mtime'],
+                        created_on=entry['ctime'])
+            search.index_wiki(name, body)

realms/modules/search/hooks.py

@@ -1,15 +1,21 @@
 from realms.modules.wiki.models import Wiki
-from realms.modules.search.models import Search
+from realms import search
+
 
 
 @Wiki.after('write_page')
 def wiki_write_page(name, content, message=None, username=None, email=None, **kwargs):
+
+    if not hasattr(search, 'index_wiki'):
+        # using simple search or none
+        return
+
     body = dict(name=name,
                 content=content,
                 message=message,
                 email=email,
                 username=username)
-    return Search.index('wiki', 'page', id_=name, body=body)
+    return search.index_wiki(name, body)
 
 
 @Wiki.after('rename_page')

realms/modules/search/models.py

@@ -1,26 +1,71 @@
-from realms import elastic
+from flask import g, current_app
-from realms.lib.model import HookMixin
+from realms.lib.util import filename_to_cname
 
 
-class Search(HookMixin):
+def simple(app):
+    return SimpleSearch()
 
-    @classmethod
-    def index(cls, index, doc_type, id_=None, body=None):
-        return elastic.index(index=index, doc_type=doc_type, id=id_, body=body)
 
-    @classmethod
+def elasticsearch(app):
-    def wiki(cls, query):
+    from flask.ext.elastic import Elastic
+    return ElasticSearch(Elastic(app))
+
+
+class Search(object):
+    def __init__(self, app=None):
+        if app is not None:
+            self.init_app(app)
+
+    def init_app(self, app):
+        search_obj = globals()[app.config['SEARCH_TYPE']]
+        app.extensions['search'] = search_obj(app)
+
+    def __getattr__(self, item):
+        return getattr(current_app.extensions['search'], item)
+
+
+class BaseSearch():
+    pass
+
+
+class SimpleSearch(BaseSearch):
+    def wiki(self, query):
+        res = []
+        for entry in g.current_wiki.get_index():
+            name = filename_to_cname(entry['name'])
+            if query in name.split('-'):
+                page = g.current_wiki.get_page(name)
+                res.append(dict(name=name, content=page['data']))
+        return res
+
+    def users(self, query):
+        pass
+
+
+class ElasticSearch(BaseSearch):
+    def __init__(self, elastic):
+        self.elastic = elastic
+
+    def index(self, index, doc_type, id_=None, body=None):
+        return self.elastic.index(index=index, doc_type=doc_type, id=id_, body=body)
+
+    def index_wiki(self, name, body):
+        self.index('wiki', 'page', id_=name, body=body)
+
+    def delete_index(self, index):
+        return self.elastic.indices.delete(index=index, ignore=[400, 404])
+
+    def wiki(self, query):
         if not query:
             return []
 
-        res = elastic.search(index='wiki', body={"query": {
+        res = self.elastic.search(index='wiki', body={"query": {
             "multi_match": {
                 "query": query,
-                "fields": ["name^3", "content"]
+                "fields": ["name"]
             }}})
 
         return [hit["_source"] for hit in res['hits']['hits']]
 
-    @classmethod
+    def users(self, query):
-    def users(cls, query):
         pass

realms/modules/search/views.py

@@ -1,10 +1,10 @@
 from flask import abort, g, render_template, request, redirect, Blueprint, flash, url_for, current_app
-from .models import Search
+from realms import search as search_engine
 
 blueprint = Blueprint('search', __name__)
 
 
 @blueprint.route('/_search')
 def search():
-    results = Search.wiki(request.args.get('q'))
+    results = search_engine.wiki(request.args.get('q'))
     return render_template('search/search.html', results=results)

realms/modules/wiki/models.py

@@ -5,32 +5,11 @@ import gittle.utils
 import yaml
 from gittle import Gittle
 from dulwich.repo import NotGitRepository
-from realms.lib.util import to_canonical
+from realms.lib.util import to_canonical, cname_to_filename, filename_to_cname
 from realms import cache
 from realms.lib.hook import HookMixin
 
 
-def cname_to_filename(cname):
-    """ Convert canonical name to filename
-
-    :param cname: Canonical name
-    :return: str -- Filename
-
-    """
-    return cname + ".md"
-
-
-def filename_to_cname(filename):
-    """Convert filename to canonical name.
-
-    .. note::
-
-    It's assumed filename is already canonical format
-
-    """
-    return os.path.splitext(filename)[0]
-
-
 class PageNotFound(Exception):
     pass
 

realms/modules/wiki/tests.py

@@ -1,7 +1,7 @@
 import json
 from nose.tools import *
 from flask import url_for
-from realms.modules.wiki.models import cname_to_filename, filename_to_cname
+from realms.lib.util import cname_to_filename, filename_to_cname
 from realms.lib.test import BaseTest