diff --git a/install.sh b/install.sh index 9660b79..5fb737f 100755 --- a/install.sh +++ b/install.sh @@ -10,13 +10,28 @@ if [ -d "/vagrant" ]; then fi echo "Provisioning..." -sudo apt-get update -sudo apt-get install -y software-properties-common python-software-properties + +if ! type "add-apt-repository" > /dev/null; then + sudo apt-get update + sudo apt-get install -y software-properties-common python-software-properties +fi + +# Elastic Search +wget -qO - http://packages.elasticsearch.org/GPG-KEY-elasticsearch | sudo apt-key add - +echo 'deb http://packages.elasticsearch.org/elasticsearch/1.4/debian stable main' | sudo tee /etc/apt/sources.list.d/elastic.list + sudo add-apt-repository -y ppa:chris-lea/node.js sudo apt-get update + sudo apt-get install -y python build-essential pkg-config git \ python-pip python-virtualenv python-dev zlib1g-dev \ -libffi-dev libyaml-dev libssl-dev nodejs +libffi-dev libyaml-dev libssl-dev nodejs openjdk-7-jre-headless elasticsearch + +# Create swap file because ES eats up RAM and 14.04 doesn't have swap by default +sudo fallocate -l 1G /swapfile +sudo chmod 600 /swapfile +sudo mkswap /swapfile +sudo swapon /swapfile # lxml deps # libxml2-dev libxslt1-dev @@ -63,4 +78,6 @@ sudo mv /tmp/realms-wiki /usr/local/bin sudo chmod +x /usr/local/bin/realms-wiki +sudo service elasticsearch start + realms-wiki start \ No newline at end of file diff --git a/realms/__init__.py b/realms/__init__.py index f550e3a..19160c9 100644 --- a/realms/__init__.py +++ b/realms/__init__.py @@ -19,6 +19,7 @@ from werkzeug.routing import BaseConverter from werkzeug.exceptions import HTTPException from sqlalchemy.ext.declarative import declarative_base +from .modules.search.models import Search from .lib.util import to_canonical, remove_ext, mkdir_safe, gravatar_url, to_dict from .lib.hook import HookModelMeta, HookMixin from .lib.util import is_su, in_virtualenv @@ -161,6 +162,7 @@ def create_app(config=None): db.init_app(app) cache.init_app(app) assets.init_app(app) + search.init_app(app) db.Model = declarative_base(metaclass=HookModelMeta, cls=HookMixin) @@ -199,6 +201,7 @@ login_manager = LoginManager() db = SQLAlchemy() cache = Cache() assets = Assets() +search = Search() assets.register('main.js', 'vendor/jquery/dist/jquery.js', diff --git a/realms/config/__init__.py b/realms/config/__init__.py index 8f8eedb..d686686 100644 --- a/realms/config/__init__.py +++ b/realms/config/__init__.py @@ -95,6 +95,11 @@ CACHE_REDIS_DB = '0' #CACHE_TYPE = 'memcached' CACHE_MEMCACHED_SERVERS = ['127.0.0.1:11211'] +SEARCH_TYPE = 'simple' # simple is not good for large wikis + +# SEARCH_TYPE = 'elasticsearch' +ELASTICSEARCH_URL = 'http://127.0.0.1:9200' + # Get ReCaptcha Keys for your domain here: # https://www.google.com/recaptcha/admin#whyrecaptcha RECAPTCHA_ENABLE = False @@ -146,4 +151,4 @@ if ENV != "DEV": ASSETS_DEBUG = False SQLALCHEMY_ECHO = False -MODULES = ['wiki', 'auth'] +MODULES = ['wiki', 'auth', 'search'] diff --git a/realms/lib/hook.py b/realms/lib/hook.py index e772a1f..bf6a00e 100644 --- a/realms/lib/hook.py +++ b/realms/lib/hook.py @@ -7,12 +7,15 @@ def hook_func(name, fn): @wraps(fn) def wrapper(self, *args, **kwargs): for hook, a, kw in self.__class__._pre_hooks.get(name) or []: - hook(*a, **kw) + hook(*args, **kwargs) rv = fn(self, *args, **kwargs) + # Attach return value for post hooks + kwargs.update(dict(rv=rv)) + for hook, a, kw in self.__class__._post_hooks.get(name) or []: - hook(*a, **kw) + hook(*args, **kwargs) return rv return wrapper diff --git a/realms/lib/util.py b/realms/lib/util.py index d3e7656..a707805 100644 --- a/realms/lib/util.py +++ b/realms/lib/util.py @@ -97,6 +97,25 @@ def to_canonical(s): s = s.lower() return s +def cname_to_filename(cname): + """ Convert canonical name to filename + + :param cname: Canonical name + :return: str -- Filename + + """ + return cname + ".md" + + +def filename_to_cname(filename): + """Convert filename to canonical name. + + .. note:: + + It's assumed filename is already canonical format + + """ + return os.path.splitext(filename)[0] def gravatar_url(email): return "//www.gravatar.com/avatar/" + hashlib.md5(email).hexdigest() diff --git a/realms/modules/search/__init__.py b/realms/modules/search/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/realms/modules/search/commands.py b/realms/modules/search/commands.py new file mode 100644 index 0000000..6c981ec --- /dev/null +++ b/realms/modules/search/commands.py @@ -0,0 +1,36 @@ +import click +from realms import create_app, search +from realms.modules.wiki.models import Wiki +from realms.lib.util import filename_to_cname + + +@click.group(short_help="Search Module") +def cli(): + pass + + +@cli.command() +def rebuild_index(): + """ Rebuild search index + """ + app = create_app() + + if app.config.get('SEARCH_TYPE') == 'simple': + click.echo("Search type is simple, try using elasticsearch.") + return + + with app.app_context(): + # Wiki + search.delete_index('wiki') + wiki = Wiki(app.config['WIKI_PATH']) + for entry in wiki.get_index(): + page = wiki.get_page(entry['name']) + name = filename_to_cname(page['name']) + # TODO add email? + body = dict(name=name, + content=page['data'], + message=page['info']['message'], + username=page['info']['author'], + updated_on=entry['mtime'], + created_on=entry['ctime']) + search.index_wiki(name, body) \ No newline at end of file diff --git a/realms/modules/search/hooks.py b/realms/modules/search/hooks.py new file mode 100644 index 0000000..847fee9 --- /dev/null +++ b/realms/modules/search/hooks.py @@ -0,0 +1,23 @@ +from realms.modules.wiki.models import Wiki +from realms import search + + + +@Wiki.after('write_page') +def wiki_write_page(name, content, message=None, username=None, email=None, **kwargs): + + if not hasattr(search, 'index_wiki'): + # using simple search or none + return + + body = dict(name=name, + content=content, + message=message, + email=email, + username=username) + return search.index_wiki(name, body) + + +@Wiki.after('rename_page') +def wiki_rename_page(*args, **kwargs): + pass \ No newline at end of file diff --git a/realms/modules/search/models.py b/realms/modules/search/models.py new file mode 100644 index 0000000..eacce27 --- /dev/null +++ b/realms/modules/search/models.py @@ -0,0 +1,71 @@ +from flask import g, current_app +from realms.lib.util import filename_to_cname + + +def simple(app): + return SimpleSearch() + + +def elasticsearch(app): + from flask.ext.elastic import Elastic + return ElasticSearch(Elastic(app)) + + +class Search(object): + def __init__(self, app=None): + if app is not None: + self.init_app(app) + + def init_app(self, app): + search_obj = globals()[app.config['SEARCH_TYPE']] + app.extensions['search'] = search_obj(app) + + def __getattr__(self, item): + return getattr(current_app.extensions['search'], item) + + +class BaseSearch(): + pass + + +class SimpleSearch(BaseSearch): + def wiki(self, query): + res = [] + for entry in g.current_wiki.get_index(): + name = filename_to_cname(entry['name']) + if query in name.split('-'): + page = g.current_wiki.get_page(name) + res.append(dict(name=name, content=page['data'])) + return res + + def users(self, query): + pass + + +class ElasticSearch(BaseSearch): + def __init__(self, elastic): + self.elastic = elastic + + def index(self, index, doc_type, id_=None, body=None): + return self.elastic.index(index=index, doc_type=doc_type, id=id_, body=body) + + def index_wiki(self, name, body): + self.index('wiki', 'page', id_=name, body=body) + + def delete_index(self, index): + return self.elastic.indices.delete(index=index, ignore=[400, 404]) + + def wiki(self, query): + if not query: + return [] + + res = self.elastic.search(index='wiki', body={"query": { + "multi_match": { + "query": query, + "fields": ["name"] + }}}) + + return [hit["_source"] for hit in res['hits']['hits']] + + def users(self, query): + pass diff --git a/realms/modules/search/views.py b/realms/modules/search/views.py new file mode 100644 index 0000000..5357392 --- /dev/null +++ b/realms/modules/search/views.py @@ -0,0 +1,10 @@ +from flask import abort, g, render_template, request, redirect, Blueprint, flash, url_for, current_app +from realms import search as search_engine + +blueprint = Blueprint('search', __name__) + + +@blueprint.route('/_search') +def search(): + results = search_engine.wiki(request.args.get('q')) + return render_template('search/search.html', results=results) \ No newline at end of file diff --git a/realms/modules/wiki/models.py b/realms/modules/wiki/models.py index a513755..8d7a310 100644 --- a/realms/modules/wiki/models.py +++ b/realms/modules/wiki/models.py @@ -5,32 +5,11 @@ import gittle.utils import yaml from gittle import Gittle from dulwich.repo import NotGitRepository -from realms.lib.util import to_canonical +from realms.lib.util import to_canonical, cname_to_filename, filename_to_cname from realms import cache from realms.lib.hook import HookMixin -def cname_to_filename(cname): - """ Convert canonical name to filename - - :param cname: Canonical name - :return: str -- Filename - - """ - return cname + ".md" - - -def filename_to_cname(filename): - """Convert filename to canonical name. - - .. note:: - - It's assumed filename is already canonical format - - """ - return os.path.splitext(filename)[0] - - class PageNotFound(Exception): pass @@ -124,7 +103,6 @@ class Wiki(HookMixin): return ret - def rename_page(self, old_name, new_name, username=None, email=None, message=None): """Rename page. diff --git a/realms/modules/wiki/tests.py b/realms/modules/wiki/tests.py index 1cf3f5d..b405103 100644 --- a/realms/modules/wiki/tests.py +++ b/realms/modules/wiki/tests.py @@ -1,7 +1,7 @@ import json from nose.tools import * from flask import url_for -from realms.modules.wiki.models import cname_to_filename, filename_to_cname +from realms.lib.util import cname_to_filename, filename_to_cname from realms.lib.test import BaseTest diff --git a/realms/static/css/style.css b/realms/static/css/style.css index 0709810..1116438 100644 --- a/realms/static/css/style.css +++ b/realms/static/css/style.css @@ -51,6 +51,10 @@ border-radius: 0; } +.navbar .form-control { + max-height: 33px; +} + .checkbox-cell { width: 4em; padding: 0.3em; diff --git a/realms/templates/layout.html b/realms/templates/layout.html index 665ba8b..082bc74 100644 --- a/realms/templates/layout.html +++ b/realms/templates/layout.html @@ -49,7 +49,15 @@
  • History
    • {% endif %} +