From 312c61ef61e937f11994f54184994c61b08211b0 Mon Sep 17 00:00:00 2001 From: scragg Date: Mon, 10 Nov 2014 10:54:46 -0600 Subject: [PATCH 1/3] search module, wip --- Vagrantfile | 2 +- realms/__init__.py | 3 +++ realms/config/__init__.py | 4 +++- realms/lib/hook.py | 7 +++++-- realms/modules/search/__init__.py | 0 realms/modules/search/hooks.py | 15 +++++++++++++++ realms/modules/search/models.py | 17 +++++++++++++++++ realms/modules/search/views.py | 10 ++++++++++ realms/modules/wiki/models.py | 1 - realms/templates/search/search.html | 4 ++++ setup.py | 1 + 11 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 realms/modules/search/__init__.py create mode 100644 realms/modules/search/hooks.py create mode 100644 realms/modules/search/models.py create mode 100644 realms/modules/search/views.py create mode 100644 realms/templates/search/search.html diff --git a/Vagrantfile b/Vagrantfile index 56acd78..363069f 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -9,7 +9,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| vb.cpus = 2 end - config.vm.provision "shell", path: "install.sh", privileged: "false" + config.vm.provision "shell", path: "install.sh", privileged: false end Vagrant::Config.run do |config| diff --git a/realms/__init__.py b/realms/__init__.py index 08de16f..998319f 100644 --- a/realms/__init__.py +++ b/realms/__init__.py @@ -11,6 +11,7 @@ import httplib import traceback import click from flask import Flask, request, render_template, url_for, redirect, g +from flask.ext.elastic import Elastic from flask.ext.cache import Cache from flask.ext.login import LoginManager, current_user from flask.ext.sqlalchemy import SQLAlchemy, declarative_base, Model, _QueryProperty @@ -160,6 +161,7 @@ def create_app(config=None): db.init_app(app) cache.init_app(app) assets.init_app(app) + elastic.init_app(app) for status_code in httplib.responses: if status_code >= 400: @@ -197,6 +199,7 @@ login_manager = LoginManager() db = SQLAlchemy() cache = Cache() assets = Assets() +elastic = Elastic() assets.register('main.js', 'vendor/jquery/dist/jquery.js', diff --git a/realms/config/__init__.py b/realms/config/__init__.py index 8f8eedb..a961de3 100644 --- a/realms/config/__init__.py +++ b/realms/config/__init__.py @@ -95,6 +95,8 @@ CACHE_REDIS_DB = '0' #CACHE_TYPE = 'memcached' CACHE_MEMCACHED_SERVERS = ['127.0.0.1:11211'] +ELASTICSEARCH_URL = 'http://127.0.0.1:9200' + # Get ReCaptcha Keys for your domain here: # https://www.google.com/recaptcha/admin#whyrecaptcha RECAPTCHA_ENABLE = False @@ -146,4 +148,4 @@ if ENV != "DEV": ASSETS_DEBUG = False SQLALCHEMY_ECHO = False -MODULES = ['wiki', 'auth'] +MODULES = ['wiki', 'auth', 'search'] diff --git a/realms/lib/hook.py b/realms/lib/hook.py index e772a1f..bf6a00e 100644 --- a/realms/lib/hook.py +++ b/realms/lib/hook.py @@ -7,12 +7,15 @@ def hook_func(name, fn): @wraps(fn) def wrapper(self, *args, **kwargs): for hook, a, kw in self.__class__._pre_hooks.get(name) or []: - hook(*a, **kw) + hook(*args, **kwargs) rv = fn(self, *args, **kwargs) + # Attach return value for post hooks + kwargs.update(dict(rv=rv)) + for hook, a, kw in self.__class__._post_hooks.get(name) or []: - hook(*a, **kw) + hook(*args, **kwargs) return rv return wrapper diff --git a/realms/modules/search/__init__.py b/realms/modules/search/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/realms/modules/search/hooks.py b/realms/modules/search/hooks.py new file mode 100644 index 0000000..d8a7467 --- /dev/null +++ b/realms/modules/search/hooks.py @@ -0,0 +1,15 @@ +from realms.modules.wiki.models import Wiki +from realms.modules.search.models import Search + + +@Wiki.after('write_page') +def wiki_write_page(name, content, **kwargs): + body = dict(name=name, + content=content) + body.update(kwargs) + return Search.index('wiki', 'page', body=body) + + +@Wiki.after('rename_page') +def wiki_rename_page(*args, **kwargs): + pass \ No newline at end of file diff --git a/realms/modules/search/models.py b/realms/modules/search/models.py new file mode 100644 index 0000000..e0d92c1 --- /dev/null +++ b/realms/modules/search/models.py @@ -0,0 +1,17 @@ +from realms import elastic +from realms.lib.model import HookMixin + + +class Search(HookMixin): + + @classmethod + def index(cls, index, doc_type, id_=None, body=None): + return elastic.index(index=index, doc_type=doc_type, id=id_, body=body) + + @classmethod + def wiki(cls, query): + return elastic.search(index='wiki', body={"query": {"match_all": {}}}) + + @classmethod + def users(cls, query): + pass \ No newline at end of file diff --git a/realms/modules/search/views.py b/realms/modules/search/views.py new file mode 100644 index 0000000..049d849 --- /dev/null +++ b/realms/modules/search/views.py @@ -0,0 +1,10 @@ +from flask import abort, g, render_template, request, redirect, Blueprint, flash, url_for, current_app +from .models import Search +blueprint = Blueprint('search', __name__) + + +@blueprint.route('/_search') +def search(): + results = Search.wiki(request.args.get('q')) + print results + return render_template('search/search.html', results=results) \ No newline at end of file diff --git a/realms/modules/wiki/models.py b/realms/modules/wiki/models.py index a513755..e5b4ce9 100644 --- a/realms/modules/wiki/models.py +++ b/realms/modules/wiki/models.py @@ -124,7 +124,6 @@ class Wiki(HookMixin): return ret - def rename_page(self, old_name, new_name, username=None, email=None, message=None): """Rename page. diff --git a/realms/templates/search/search.html b/realms/templates/search/search.html new file mode 100644 index 0000000..38095da --- /dev/null +++ b/realms/templates/search/search.html @@ -0,0 +1,4 @@ +{% extends 'layout.html' %} +{% block body %} + +{% endblock %} diff --git a/setup.py b/setup.py index ad50832..8677160 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ setup(name='realms-wiki', 'Flask==0.10.1', 'Flask-Assets==0.10', 'Flask-Cache==0.13.1', + 'Flask-Elastic==0.2', 'Flask-Login==0.2.11', 'Flask-SQLAlchemy==2.0', 'Flask-WTF==0.10.2', From 13d4be893773e80778b6a547440f50493b0b7630 Mon Sep 17 00:00:00 2001 From: Matthew Scragg Date: Tue, 11 Nov 2014 17:06:28 -0600 Subject: [PATCH 2/3] search pass two --- install.sh | 24 ++++++++++++++++++++---- realms/modules/search/hooks.py | 10 ++++++---- realms/modules/search/models.py | 11 ++++++++++- realms/modules/search/views.py | 2 +- realms/static/css/style.css | 4 ++++ realms/templates/layout.html | 8 ++++++++ realms/templates/search/search.html | 15 ++++++++++++++- 7 files changed, 63 insertions(+), 11 deletions(-) diff --git a/install.sh b/install.sh index 5197390..5fb737f 100755 --- a/install.sh +++ b/install.sh @@ -10,13 +10,28 @@ if [ -d "/vagrant" ]; then fi echo "Provisioning..." -sudo apt-get update -sudo apt-get install -y software-properties-common python-software-properties + +if ! type "add-apt-repository" > /dev/null; then + sudo apt-get update + sudo apt-get install -y software-properties-common python-software-properties +fi + +# Elastic Search +wget -qO - http://packages.elasticsearch.org/GPG-KEY-elasticsearch | sudo apt-key add - +echo 'deb http://packages.elasticsearch.org/elasticsearch/1.4/debian stable main' | sudo tee /etc/apt/sources.list.d/elastic.list + sudo add-apt-repository -y ppa:chris-lea/node.js sudo apt-get update + sudo apt-get install -y python build-essential pkg-config git \ python-pip python-virtualenv python-dev zlib1g-dev \ -libffi-dev libyaml-dev libssl-dev nodejs +libffi-dev libyaml-dev libssl-dev nodejs openjdk-7-jre-headless elasticsearch + +# Create swap file because ES eats up RAM and 14.04 doesn't have swap by default +sudo fallocate -l 1G /swapfile +sudo chmod 600 /swapfile +sudo mkswap /swapfile +sudo swapon /swapfile # lxml deps # libxml2-dev libxslt1-dev @@ -45,7 +60,6 @@ sudo npm install -g bower cd /home/vagrant - virtualenv .venv source .venv/bin/activate @@ -64,4 +78,6 @@ sudo mv /tmp/realms-wiki /usr/local/bin sudo chmod +x /usr/local/bin/realms-wiki +sudo service elasticsearch start + realms-wiki start \ No newline at end of file diff --git a/realms/modules/search/hooks.py b/realms/modules/search/hooks.py index d8a7467..1c682c2 100644 --- a/realms/modules/search/hooks.py +++ b/realms/modules/search/hooks.py @@ -3,11 +3,13 @@ from realms.modules.search.models import Search @Wiki.after('write_page') -def wiki_write_page(name, content, **kwargs): +def wiki_write_page(name, content, message=None, username=None, email=None, **kwargs): body = dict(name=name, - content=content) - body.update(kwargs) - return Search.index('wiki', 'page', body=body) + content=content, + message=message, + email=email, + username=username) + return Search.index('wiki', 'page', id_=name, body=body) @Wiki.after('rename_page') diff --git a/realms/modules/search/models.py b/realms/modules/search/models.py index e0d92c1..c0ea692 100644 --- a/realms/modules/search/models.py +++ b/realms/modules/search/models.py @@ -10,7 +10,16 @@ class Search(HookMixin): @classmethod def wiki(cls, query): - return elastic.search(index='wiki', body={"query": {"match_all": {}}}) + if not query: + return [] + + res = elastic.search(index='wiki', body={"query": { + "multi_match": { + "query": query, + "fields": ["name^3", "content"] + }}}) + + return [hit["_source"] for hit in res['hits']['hits']] @classmethod def users(cls, query): diff --git a/realms/modules/search/views.py b/realms/modules/search/views.py index 049d849..f08068f 100644 --- a/realms/modules/search/views.py +++ b/realms/modules/search/views.py @@ -1,10 +1,10 @@ from flask import abort, g, render_template, request, redirect, Blueprint, flash, url_for, current_app from .models import Search + blueprint = Blueprint('search', __name__) @blueprint.route('/_search') def search(): results = Search.wiki(request.args.get('q')) - print results return render_template('search/search.html', results=results) \ No newline at end of file diff --git a/realms/static/css/style.css b/realms/static/css/style.css index 0709810..1116438 100644 --- a/realms/static/css/style.css +++ b/realms/static/css/style.css @@ -51,6 +51,10 @@ border-radius: 0; } +.navbar .form-control { + max-height: 33px; +} + .checkbox-cell { width: 4em; padding: 0.3em; diff --git a/realms/templates/layout.html b/realms/templates/layout.html index 665ba8b..082bc74 100644 --- a/realms/templates/layout.html +++ b/realms/templates/layout.html @@ -49,7 +49,15 @@
  • History
    • {% endif %} +