From 02b6b7d592ad62ccc63ad4c31c385bb15712ff0e Mon Sep 17 00:00:00 2001 From: Matthew Scragg Date: Mon, 2 Dec 2013 17:50:19 -0600 Subject: [PATCH] stuff --- app.py | 9 +- realms/__init__.py | 484 ++++++++++++++++++------------------- realms/lib/services.py | 7 +- realms/lib/wiki.py | 6 +- realms/models.py | 54 ++--- requirements.txt | 12 +- srv/salt/postgres/init.sls | 2 + 7 files changed, 273 insertions(+), 301 deletions(-) diff --git a/app.py b/app.py index 9942101..12f2880 100644 --- a/app.py +++ b/app.py @@ -1,9 +1,10 @@ from gevent import monkey, pywsgi monkey.patch_all() -from realms import config, init_db, make_app, SubdomainDispatcher +import logging +from realms import app, config if __name__ == '__main__': - init_db(config.db['dbname']) - app = SubdomainDispatcher(config.domain, make_app) - pywsgi.WSGIServer(('', config.port), app).serve_forever() + print "Starting server" + app.logger.setLevel(logging.INFO) + pywsgi.WSGIServer(('', config.PORT), app).serve_forever() diff --git a/realms/__init__.py b/realms/__init__.py index 32f4d64..9a21be5 100644 --- a/realms/__init__.py +++ b/realms/__init__.py @@ -1,10 +1,9 @@ import logging import os import time -from threading import Lock +from tldextract import tldextract -import rethinkdb as rdb -from flask import Flask, g, request, render_template, url_for, redirect, flash, session +from flask import Flask, g, request, render_template, url_for, redirect, flash, session, current_app from flask.ctx import _AppCtxGlobals from flask.ext.login import LoginManager, login_required from flask.ext.assets import Environment, Bundle @@ -20,16 +19,14 @@ from realms.lib.services import db from models import Site, User, CurrentUser -# Flask instance container -instances = {} - -# Flask extension objects -login_manager = LoginManager() -assets = Environment() - - class AppCtxGlobals(_AppCtxGlobals): + @cached_property + def current_site(self): + ext = tldextract.extract(request.host) + print ext + return ext.subdomain + @cached_property def current_user(self): return session.get('user') if session.get('user') else {'username': 'Anon'} @@ -38,54 +35,30 @@ class AppCtxGlobals(_AppCtxGlobals): class Application(Flask): app_ctx_globals_class = AppCtxGlobals - -class SubdomainDispatcher(object): - """ - Application factory - """ - def __init__(self, domain, create_app): - self.domain = domain - self.create_app = create_app - self.lock = Lock() - - def get_application(self, host): - host = host.split(':')[0] - assert host.endswith(self.domain), 'Configuration error' - subdomain = host[:-len(self.domain)].rstrip('.') - with self.lock: - app = instances.get(subdomain) - if app is None: - app = self.create_app(subdomain) - instances[subdomain] = app - return app - def __call__(self, environ, start_response): - app = self.get_application(environ['HTTP_HOST']) - return app(environ, start_response) + path_info = environ.get('PATH_INFO') + + if path_info and len(path_info) > 1 and path_info.endswith('/'): + environ['PATH_INFO'] = path_info[:-1] + + scheme = environ.get('HTTP_X_SCHEME') + + if scheme: + environ['wsgi.url_scheme'] = scheme + + real_ip = environ.get('HTTP_X_REAL_IP') + + if real_ip: + environ['REMOTE_ADDR'] = real_ip + + return super(Application, self).__call__(environ, start_response) def init_db(dbname): """ Assures DB has minimal setup """ - if not dbname in rdb.db_list().run(db): - print "Creating DB %s" % dbname - rdb.db_create(dbname).run(db) - - for tbl in ['sites', 'users', 'pages']: - if not tbl in rdb.table_list().run(db): - rdb.table_create(tbl).run(db) - - if not 'name' in rdb.table('sites').index_list().run(db): - rdb.table('sites').index_create('name').run(db) - - for i in ['username', 'email']: - if not i in rdb.table('users').index_list().run(db): - rdb.table('users').index_create(i).run(db) - - s = Site() - if not s.get_by_name('_'): - s.create(name='_', repo='_') + pass class RegexConverter(BaseConverter): @@ -114,229 +87,240 @@ def format_subdomain(s): return s -def make_app(subdomain): - subdomain = format_subdomain(subdomain) - if subdomain and not Wiki.is_registered(subdomain): - return redirect("http://%s/_new/?site=%s" % (config.hostname, subdomain)) - return create_app(subdomain) +app = Application(__name__) +app.config.update(config.FLASK) +app.debug = (config.ENV is not 'PROD') +app.secret_key = config.SECRET_KEY +app.static_path = os.sep + 'static' +app.session_interface = RedisSessionInterface() +app.url_map.converters['regex'] = RegexConverter + +# Flask extension objects +login_manager = LoginManager() +login_manager.init_app(app) +login_manager.login_view = 'login' -def create_app(subdomain=None): - app = Application(__name__) - app.config.update(config.flask) - app.debug = (config.ENV is not 'PROD') - app.secret_key = config.secret_key - app.static_path = os.sep + 'static' - app.session_interface = RedisSessionInterface() - app.url_map.converters['regex'] = RegexConverter +@login_manager.user_loader +def load_user(user_id): + return CurrentUser(user_id) - login_manager.init_app(app) - login_manager.login_view = 'login' +assets = Environment() +assets.init_app(app) +if config.ENV is 'PROD': + if 'js_common' not in assets._named_bundles: + assets.register('js_common', Bundle('packed-common.js')) + if 'js_editor' not in assets._named_bundles: + assets.register('js_editor', Bundle('packed-editor.js')) +else: + if 'js_common' not in assets._named_bundles: + js = Bundle( + Bundle('vendor/jquery/jquery.js', + 'vendor/components-underscore/underscore.js', + 'vendor/components-bootstrap/js/bootstrap.js', + 'vendor/handlebars/handlebars.js', + 'vendor/showdown/src/showdown.js', + 'vendor/showdown/src/extensions/table.js', + 'js/wmd.js', + filters='closure_js'), + 'js/html-sanitizer-minified.js', + 'vendor/highlightjs/highlight.pack.js', + Bundle('js/main.js', filters='closure_js'), + output='packed-common.js') + assets.register('js_common', js) - @login_manager.user_loader - def load_user(user_id): - return CurrentUser(user_id) + if 'js_editor' not in assets._named_bundles: + js = Bundle('js/ace/ace.js', + 'js/ace/mode-markdown.js', + 'vendor/keymaster/keymaster.js', + 'js/dillinger.js', + filters='closure_js', output='packed-editor.js') + assets.register('js_editor', js) - assets.init_app(app) - if config.ENV is 'PROD': - if 'js_common' not in assets._named_bundles: - assets.register('js_common', Bundle('packed-common.js')) - if 'js_editor' not in assets._named_bundles: - assets.register('js_editor', Bundle('packed-editor.js')) +repo_dir = config.REPO_DIR + + +@app.after_request +def inject_x_rate_headers(response): + limit = get_view_rate_limit() + if limit and limit.send_x_headers: + h = response.headers + h.add('X-RateLimit-Remaining', str(limit.remaining)) + h.add('X-RateLimit-Limit', str(limit.limit)) + h.add('X-RateLimit-Reset', str(limit.reset)) + return response + + +@app.template_filter('datetime') +def _jinja2_filter_datetime(ts): + return time.strftime('%b %d, %Y %I:%M %p', time.localtime(ts)) + +@app.errorhandler(404) +def page_not_found(e): + return render_template('errors/404.html'), 404 + + +@app.errorhandler(500) +def page_error(e): + logging.exception(e) + return render_template('errors/500.html'), 500 + + +@app.route("/") +@ratelimiter(limit=50, per=60) +def root(): + return g.current_site + return render('home') + + +@app.route("/home") +def home(): + return redirect(url_for('root')) + + +@app.route("/_account/") +@login_required +def account(): + return render_template('account/index.html') + + +@app.route("/_new/", methods=['GET', 'POST']) +@login_required +def new_wiki(): + if request.method == 'POST': + wiki_name = to_canonical(request.form['name']) + + if Wiki.is_registered(wiki_name): + flash("Site already exists") + return redirect(redirect_url()) + else: + s = Site() + s.create(name=wiki_name, repo=wiki_name, founder=g.current_user.get('id')) + return redirect('http://%s.%s' % (wiki_name, config.hostname)) else: - if 'js_common' not in assets._named_bundles: - js = Bundle( - Bundle('vendor/jquery/jquery.js', - 'vendor/components-underscore/underscore.js', - 'vendor/components-bootstrap/js/bootstrap.js', - 'vendor/handlebars/handlebars.js', - 'vendor/showdown/src/showdown.js', - 'vendor/showdown/src/extensions/table.js', - 'js/wmd.js', - filters='closure_js'), - 'js/html-sanitizer-minified.js', - 'vendor/highlightjs/highlight.pack.js', - Bundle('js/main.js', filters='closure_js'), - output='packed-common.js') - assets.register('js_common', js) + return render_template('_new/index.html') - if 'js_editor' not in assets._named_bundles: - js = Bundle('js/ace/ace.js', - 'js/ace/mode-markdown.js', - 'vendor/keymaster/keymaster.js', - 'js/dillinger.js', - filters='closure_js', output='packed-editor.js') - assets.register('js_editor', js) - repo_dir = config.repos['dir'] - repo_name = subdomain if subdomain else "_" +@app.route("/_logout/") +def logout(): + User.logout() + return redirect(url_for('root')) - w = Wiki(repo_dir + "/" + repo_name) - @app.after_request - def inject_x_rate_headers(response): - limit = get_view_rate_limit() - if limit and limit.send_x_headers: - h = response.headers - h.add('X-RateLimit-Remaining', str(limit.remaining)) - h.add('X-RateLimit-Limit', str(limit.limit)) - h.add('X-RateLimit-Reset', str(limit.reset)) - return response +@app.route("/_commit//") +def commit_sha(name, sha): + cname = to_canonical(name) - @app.template_filter('datetime') - def _jinja2_filter_datetime(ts): - return time.strftime('%b %d, %Y %I:%M %p', time.localtime(ts)) + data = Wiki.get_page(cname, sha=sha) + if data: + return render_template('page/page.html', name=name, page=data, commit=sha) + else: + return redirect('/_create/'+cname) - @app.errorhandler(404) - def page_not_found(e): - return render_template('errors/404.html'), 404 - @app.errorhandler(500) - def page_error(e): - logging.exception(e) - return render_template('errors/500.html'), 500 +@app.route("/_compare//") +def compare(name, fsha, dots, lsha): + diff = Wiki.compare(name, fsha, lsha) + return render_template('page/compare.html', name=name, diff=diff, old=fsha, new=lsha) - @app.route("/") - @ratelimiter(limit=50, per=60) - def root(): - return render('home') - @app.route("/home") - def home(): - return redirect(url_for('root')) - - @app.route("/_account/") - @login_required - def account(): - return render_template('account/index.html') - - @app.route("/_new/", methods=['GET', 'POST']) - @login_required - def new_wiki(): - if request.method == 'POST': - wiki_name = to_canonical(request.form['name']) - - if Wiki.is_registered(wiki_name): - flash("Site already exists") - return redirect(redirect_url()) - else: - s = Site() - s.create(name=wiki_name, repo=wiki_name, founder=g.current_user.get('id')) - instances.pop(wiki_name, None) - return redirect('http://%s.%s' % (wiki_name, config.hostname)) - else: - return render_template('_new/index.html') - - @app.route("/_logout/") - def logout(): - User.logout() - return redirect(url_for('root')) - - @app.route("/_commit//") - def commit_sha(name, sha): +@app.route("/_revert", methods=['POST']) +def revert(): + if request.method == 'POST': + name = request.form.get('name') + commit = request.form.get('commit') cname = to_canonical(name) + Wiki.revert_page(name, commit, message="Reverting %s" % cname, username=g.current_user.get('username')) + flash('Page reverted', 'success') + return redirect("/" + cname) - data = w.get_page(cname, sha=sha) + +@app.route("/_register", methods=['GET', 'POST']) +def register(): + if request.method == 'POST': + if User.register(request.form.get('username'), request.form.get('email'), request.form.get('password')): + return redirect(url_for('root')) + else: + # Login failed + return redirect(url_for('register')) + else: + return render_template('account/register.html') + + +@app.route("/_login", methods=['GET', 'POST']) +def login(): + if request.method == 'POST': + if User.auth(request.form['email'], request.form['password']): + return redirect(redirect_url(referrer=url_for('root'))) + else: + flash("Email or Password invalid") + return redirect("/_login") + else: + return render_template('account/login.html') + + +@app.route("/_history/") +def history(name): + history = Wiki.get_history(name) + return render_template('page/history.html', name=name, history=history) + + +@app.route("/_edit/", methods=['GET', 'POST']) +def edit(name): + data = Wiki.get_page(name) + cname = to_canonical(name) + if request.method == 'POST': + edit_cname = to_canonical(request.form['name']) + if edit_cname.lower() != cname.lower(): + Wiki.rename_page(cname, edit_cname) + Wiki.write_page(edit_cname, + request.form['content'], + message=request.form['message'], + username=g.current_user.get('username')) + return redirect("/" + edit_cname) + else: if data: - return render_template('page/page.html', name=name, page=data, commit=sha) + name = remove_ext(data['name']) + content = data['data'] + return render_template('page/edit.html', name=name, content=content) else: return redirect('/_create/'+cname) - @app.route("/_compare//") - def compare(name, fsha, dots, lsha): - diff = w.compare(name, fsha, lsha) - return render_template('page/compare.html', name=name, diff=diff, old=fsha, new=lsha) - @app.route("/_revert", methods=['POST']) - def revert(): - if request.method == 'POST': - name = request.form.get('name') - commit = request.form.get('commit') - cname = to_canonical(name) - w.revert_page(name, commit, message="Reverting %s" % cname, username=g.current_user.get('username')) - flash('Page reverted', 'success') - return redirect("/" + cname) +@app.route("/_delete/", methods=['POST']) +@login_required +def delete(name): + pass - @app.route("/_register", methods=['GET', 'POST']) - def register(): - if request.method == 'POST': - if User.register(request.form.get('username'), request.form.get('email'), request.form.get('password')): - return redirect(url_for('root')) - else: - # Login failed - return redirect(url_for('register')) - else: - return render_template('account/register.html') - @app.route("/_login", methods=['GET', 'POST']) - def login(): - if request.method == 'POST': - if User.auth(request.form['email'], request.form['password']): - return redirect(redirect_url(referrer=url_for('root'))) - else: - flash("Email or Password invalid") - return redirect("/_login") - else: - return render_template('account/login.html') - - @app.route("/_history/") - def history(name): - history = w.get_history(name) - return render_template('page/history.html', name=name, history=history) - - @app.route("/_edit/", methods=['GET', 'POST']) - def edit(name): - data = w.get_page(name) +@app.route("/_create/", methods=['GET', 'POST']) +@app.route("/_create/", methods=['GET', 'POST']) +def create(name=None): + cname = "" + if name: cname = to_canonical(name) - if request.method == 'POST': - edit_cname = to_canonical(request.form['name']) - if edit_cname.lower() != cname.lower(): - w.rename_page(cname, edit_cname) - w.write_page(edit_cname, request.form['content'], - message=request.form['message'], - username=g.current_user.get('username')) - return redirect("/" + edit_cname) - else: - if data: - name = remove_ext(data['name']) - content = data['data'] - return render_template('page/edit.html', name=name, content=content) - else: - return redirect('/_create/'+cname) + if Wiki.get_page(cname): + # Page exists, edit instead + return redirect("/edit/" + cname) + if request.method == 'POST': + Wiki.write_page(request.form['name'], + request.form['content'], + message=request.form['message'], + create=True, + username=g.current_user.get('username')) + return redirect("/" + cname) + else: + return render_template('page/edit.html', name=cname, content="") - @app.route("/_delete/", methods=['POST']) - @login_required - def delete(name): - pass - @app.route("/_create/", methods=['GET', 'POST']) - @app.route("/_create/", methods=['GET', 'POST']) - def create(name=None): - cname = "" - if name: - cname = to_canonical(name) - if w.get_page(cname): - # Page exists, edit instead - return redirect("/edit/" + cname) - if request.method == 'POST': - w.write_page(request.form['name'], request.form['content'], - message=request.form['message'], - create=True, - username=g.current_user.get('username')) - return redirect("/" + cname) - else: - return render_template('page/edit.html', name=cname, content="") +@app.route("/") +def render(name): + cname = to_canonical(name) + if cname != name: + return redirect('/' + cname) - @app.route("/") - def render(name): - cname = to_canonical(name) - if cname != name: - return redirect('/' + cname) - - data = w.get_page(cname) - if data: - return render_template('page/page.html', name=cname, page=data) - else: - return redirect('/_create/'+cname) - - return app \ No newline at end of file + data = Wiki.get_page(cname) + if data: + return render_template('page/page.html', name=cname, page=data) + else: + return redirect('/_create/'+cname) diff --git a/realms/lib/services.py b/realms/lib/services.py index 0ad57c4..0069cce 100644 --- a/realms/lib/services.py +++ b/realms/lib/services.py @@ -1,10 +1,9 @@ -import rethinkdb as rdb import redis from realms import config - +from sqlalchemy import create_engine # Default DB connection -db = rdb.connect(config.db['host'], config.db['port'], db=config.db['dbname']) +db = create_engine(config.DB_URI, encoding='utf8', echo=True) # Default Cache connection -cache = redis.StrictRedis(host=config.cache['host'], port=config.cache['port']) \ No newline at end of file +cache = redis.StrictRedis(host=config.REDIS_HOST, port=config.REDIS_PORT) \ No newline at end of file diff --git a/realms/lib/wiki.py b/realms/lib/wiki.py index 50353b2..634887e 100644 --- a/realms/lib/wiki.py +++ b/realms/lib/wiki.py @@ -1,15 +1,13 @@ import os import re import lxml.html -from lxml.html import clean import ghdiff - import gittle.utils from gittle import Gittle from dulwich.repo import NotGitRepository from werkzeug.utils import escape, unescape from util import to_canonical -from models import Site +from realms.models import Site class MyGittle(Gittle): @@ -94,7 +92,7 @@ class Wiki(): tree = lxml.html.fromstring(content) - cleaner = clean.Cleaner(remove_unknown_tags=False, kill_tags=set(['style']), safe_attrs_only=False) + cleaner = lxml.html.Cleaner(remove_unknown_tags=False, kill_tags=set(['style']), safe_attrs_only=False) tree = cleaner.clean_html(tree) content = lxml.html.tostring(tree, encoding='utf-8', method='html') diff --git a/realms/models.py b/realms/models.py index c15808e..aeb6a9e 100644 --- a/realms/models.py +++ b/realms/models.py @@ -1,40 +1,12 @@ -import rethinkdb as rdb import bcrypt +from sqlalchemy import Column, Integer, String, Time +from sqlalchemy.ext.declarative import declarative_base from flask import session, flash from flask.ext.login import login_user, logout_user -from realms.lib.util import gravatar_url, to_dict, cache_it +from realms.lib.util import gravatar_url, to_dict from realms.lib.services import db - -class BaseModel(): - - table = None - _conn = db - - def __init__(self, **kwargs): - if not kwargs.get('conn'): - kwargs['conn'] = db - - def create(self, **kwargs): - return rdb.table(self.table).insert(kwargs).run(self._conn) - - @cache_it - def get_by_id(self, id): - return rdb.table(self.table).get(id).run(self._conn) - - def get_all(self, arg, index): - return rdb.table(self.table).get_all(arg, index=index).run(self._conn) - - #@cache_it - def get_one(self, arg, index): - return rdb.table(self.table).get_all(arg, index=index).limit(1).run(self._conn) - - -class Site(BaseModel): - table = 'sites' - - def get_by_name(self, name): - return to_dict(self.get_one(name, 'name'), True) +Base = declarative_base() class CurrentUser(): @@ -66,8 +38,22 @@ class CurrentUser(): return None -class User(BaseModel): - table = 'users' +class Site(Base): + __tablename__ = 'sites' + id = Column(Integer, primary_key=True) + name = Column(String(100)) + pages = Column(Integer) + views = Column(Integer) + created = Column(Time) + + +class User(Base): + __tablename__ = 'users' + id = Column(Integer, primary_key=True) + username = Column(String(100)) + email = Column(String(255)) + password = Column(String(255)) + joined = Column(Time) def get_by_email(self, email): return to_dict(self.get_one(email, 'email'), True) diff --git a/requirements.txt b/requirements.txt index 6a0299d..31346d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,21 +3,23 @@ Flask-Assets==0.8 Flask-Bcrypt==0.5.2 Flask-Login==0.2.7 beautifulsoup4==4.3.2 -boto==2.13.3 +boto==2.17.0 closure==20121212 gevent==0.13.8 ghdiff==0.1 gittle==0.2.2 itsdangerous==0.23 -lxml==3.2.3 +lxml==3.2.4 markdown2==2.1.0 -pyzmq==13.0.0 +pyzmq==14.0.0 recaptcha==1.0rc1 recaptcha-client==1.0.6 redis==2.8.0 rethinkdb==1.10.0-0 -simplejson==3.3.0 +simplejson==3.3.1 sockjs-tornado==1.0.0 supervisor==3.0 SQLAlchemy==0.8.3 -tornado==3.1.1 \ No newline at end of file +tornado==3.1.1 +tldextract==1.2.2 +psycopg2==2.5.1 diff --git a/srv/salt/postgres/init.sls b/srv/salt/postgres/init.sls index 081a298..a70b192 100644 --- a/srv/salt/postgres/init.sls +++ b/srv/salt/postgres/init.sls @@ -2,6 +2,8 @@ postgresql: pkg.installed: - name: postgresql-9.3 +libpq-dev: + pkg.installed createdb: cmd.run: