diff --git a/realms/__init__.py b/realms/__init__.py index 998319f..52e126f 100644 --- a/realms/__init__.py +++ b/realms/__init__.py @@ -11,14 +11,14 @@ import httplib import traceback import click from flask import Flask, request, render_template, url_for, redirect, g -from flask.ext.elastic import Elastic from flask.ext.cache import Cache from flask.ext.login import LoginManager, current_user -from flask.ext.sqlalchemy import SQLAlchemy, declarative_base, Model, _QueryProperty +from flask.ext.sqlalchemy import SQLAlchemy from flask.ext.assets import Environment, Bundle from werkzeug.routing import BaseConverter from werkzeug.exceptions import HTTPException +from .modules.search.models import Search from .lib.util import to_canonical, remove_ext, mkdir_safe, gravatar_url, to_dict from .lib.hook import HookModelMeta from .lib.util import is_su, in_virtualenv @@ -161,7 +161,7 @@ def create_app(config=None): db.init_app(app) cache.init_app(app) assets.init_app(app) - elastic.init_app(app) + search.init_app(app) for status_code in httplib.responses: if status_code >= 400: @@ -199,7 +199,7 @@ login_manager = LoginManager() db = SQLAlchemy() cache = Cache() assets = Assets() -elastic = Elastic() +search = Search() assets.register('main.js', 'vendor/jquery/dist/jquery.js', diff --git a/realms/config/__init__.py b/realms/config/__init__.py index a961de3..d686686 100644 --- a/realms/config/__init__.py +++ b/realms/config/__init__.py @@ -95,6 +95,9 @@ CACHE_REDIS_DB = '0' #CACHE_TYPE = 'memcached' CACHE_MEMCACHED_SERVERS = ['127.0.0.1:11211'] +SEARCH_TYPE = 'simple' # simple is not good for large wikis + +# SEARCH_TYPE = 'elasticsearch' ELASTICSEARCH_URL = 'http://127.0.0.1:9200' # Get ReCaptcha Keys for your domain here: diff --git a/realms/lib/util.py b/realms/lib/util.py index d3e7656..a707805 100644 --- a/realms/lib/util.py +++ b/realms/lib/util.py @@ -97,6 +97,25 @@ def to_canonical(s): s = s.lower() return s +def cname_to_filename(cname): + """ Convert canonical name to filename + + :param cname: Canonical name + :return: str -- Filename + + """ + return cname + ".md" + + +def filename_to_cname(filename): + """Convert filename to canonical name. + + .. note:: + + It's assumed filename is already canonical format + + """ + return os.path.splitext(filename)[0] def gravatar_url(email): return "//www.gravatar.com/avatar/" + hashlib.md5(email).hexdigest() diff --git a/realms/modules/search/commands.py b/realms/modules/search/commands.py new file mode 100644 index 0000000..6c981ec --- /dev/null +++ b/realms/modules/search/commands.py @@ -0,0 +1,36 @@ +import click +from realms import create_app, search +from realms.modules.wiki.models import Wiki +from realms.lib.util import filename_to_cname + + +@click.group(short_help="Search Module") +def cli(): + pass + + +@cli.command() +def rebuild_index(): + """ Rebuild search index + """ + app = create_app() + + if app.config.get('SEARCH_TYPE') == 'simple': + click.echo("Search type is simple, try using elasticsearch.") + return + + with app.app_context(): + # Wiki + search.delete_index('wiki') + wiki = Wiki(app.config['WIKI_PATH']) + for entry in wiki.get_index(): + page = wiki.get_page(entry['name']) + name = filename_to_cname(page['name']) + # TODO add email? + body = dict(name=name, + content=page['data'], + message=page['info']['message'], + username=page['info']['author'], + updated_on=entry['mtime'], + created_on=entry['ctime']) + search.index_wiki(name, body) \ No newline at end of file diff --git a/realms/modules/search/hooks.py b/realms/modules/search/hooks.py index 1c682c2..847fee9 100644 --- a/realms/modules/search/hooks.py +++ b/realms/modules/search/hooks.py @@ -1,15 +1,21 @@ from realms.modules.wiki.models import Wiki -from realms.modules.search.models import Search +from realms import search + @Wiki.after('write_page') def wiki_write_page(name, content, message=None, username=None, email=None, **kwargs): + + if not hasattr(search, 'index_wiki'): + # using simple search or none + return + body = dict(name=name, content=content, message=message, email=email, username=username) - return Search.index('wiki', 'page', id_=name, body=body) + return search.index_wiki(name, body) @Wiki.after('rename_page') diff --git a/realms/modules/search/models.py b/realms/modules/search/models.py index c0ea692..eacce27 100644 --- a/realms/modules/search/models.py +++ b/realms/modules/search/models.py @@ -1,26 +1,71 @@ -from realms import elastic -from realms.lib.model import HookMixin +from flask import g, current_app +from realms.lib.util import filename_to_cname -class Search(HookMixin): +def simple(app): + return SimpleSearch() - @classmethod - def index(cls, index, doc_type, id_=None, body=None): - return elastic.index(index=index, doc_type=doc_type, id=id_, body=body) - @classmethod - def wiki(cls, query): +def elasticsearch(app): + from flask.ext.elastic import Elastic + return ElasticSearch(Elastic(app)) + + +class Search(object): + def __init__(self, app=None): + if app is not None: + self.init_app(app) + + def init_app(self, app): + search_obj = globals()[app.config['SEARCH_TYPE']] + app.extensions['search'] = search_obj(app) + + def __getattr__(self, item): + return getattr(current_app.extensions['search'], item) + + +class BaseSearch(): + pass + + +class SimpleSearch(BaseSearch): + def wiki(self, query): + res = [] + for entry in g.current_wiki.get_index(): + name = filename_to_cname(entry['name']) + if query in name.split('-'): + page = g.current_wiki.get_page(name) + res.append(dict(name=name, content=page['data'])) + return res + + def users(self, query): + pass + + +class ElasticSearch(BaseSearch): + def __init__(self, elastic): + self.elastic = elastic + + def index(self, index, doc_type, id_=None, body=None): + return self.elastic.index(index=index, doc_type=doc_type, id=id_, body=body) + + def index_wiki(self, name, body): + self.index('wiki', 'page', id_=name, body=body) + + def delete_index(self, index): + return self.elastic.indices.delete(index=index, ignore=[400, 404]) + + def wiki(self, query): if not query: return [] - res = elastic.search(index='wiki', body={"query": { + res = self.elastic.search(index='wiki', body={"query": { "multi_match": { "query": query, - "fields": ["name^3", "content"] + "fields": ["name"] }}}) return [hit["_source"] for hit in res['hits']['hits']] - @classmethod - def users(cls, query): - pass \ No newline at end of file + def users(self, query): + pass diff --git a/realms/modules/search/views.py b/realms/modules/search/views.py index f08068f..5357392 100644 --- a/realms/modules/search/views.py +++ b/realms/modules/search/views.py @@ -1,10 +1,10 @@ from flask import abort, g, render_template, request, redirect, Blueprint, flash, url_for, current_app -from .models import Search +from realms import search as search_engine blueprint = Blueprint('search', __name__) @blueprint.route('/_search') def search(): - results = Search.wiki(request.args.get('q')) + results = search_engine.wiki(request.args.get('q')) return render_template('search/search.html', results=results) \ No newline at end of file diff --git a/realms/modules/wiki/models.py b/realms/modules/wiki/models.py index e5b4ce9..8d7a310 100644 --- a/realms/modules/wiki/models.py +++ b/realms/modules/wiki/models.py @@ -5,32 +5,11 @@ import gittle.utils import yaml from gittle import Gittle from dulwich.repo import NotGitRepository -from realms.lib.util import to_canonical +from realms.lib.util import to_canonical, cname_to_filename, filename_to_cname from realms import cache from realms.lib.hook import HookMixin -def cname_to_filename(cname): - """ Convert canonical name to filename - - :param cname: Canonical name - :return: str -- Filename - - """ - return cname + ".md" - - -def filename_to_cname(filename): - """Convert filename to canonical name. - - .. note:: - - It's assumed filename is already canonical format - - """ - return os.path.splitext(filename)[0] - - class PageNotFound(Exception): pass diff --git a/realms/modules/wiki/tests.py b/realms/modules/wiki/tests.py index 1cf3f5d..b405103 100644 --- a/realms/modules/wiki/tests.py +++ b/realms/modules/wiki/tests.py @@ -1,7 +1,7 @@ import json from nose.tools import * from flask import url_for -from realms.modules.wiki.models import cname_to_filename, filename_to_cname +from realms.lib.util import cname_to_filename, filename_to_cname from realms.lib.test import BaseTest