diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index 5c12bb8..0000000 --- a/Vagrantfile +++ /dev/null @@ -1,16 +0,0 @@ -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "precise64" - config.vm.synced_folder "srv/", "/srv/" - config.vm.provision :salt do |salt| - salt.minion_config = "srv/minion" - salt.run_highstate = true - end -end - -Vagrant::Config.run do |config| - config.vm.forward_port 80, 8000 - config.vm.forward_port 10000, 10000 - config.vm.forward_port 20000, 20000 -end diff --git a/realms/__init__.py b/realms/__init__.py index 7f7dc69..32f4d64 100644 --- a/realms/__init__.py +++ b/realms/__init__.py @@ -4,19 +4,21 @@ import time from threading import Lock import rethinkdb as rdb -from flask import Flask, request, render_template, url_for, redirect, flash +from flask import Flask, g, request, render_template, url_for, redirect, flash, session +from flask.ctx import _AppCtxGlobals from flask.ext.login import LoginManager, login_required from flask.ext.assets import Environment, Bundle -from recaptcha.client import captcha from werkzeug.routing import BaseConverter +from werkzeug.utils import cached_property -from session import RedisSessionInterface import config -from wiki import Wiki -from util import to_canonical, remove_ext, mkdir_safe, gravatar_url +from realms.lib.ratelimit import get_view_rate_limit, ratelimiter +from realms.lib.session import RedisSessionInterface +from realms.lib.wiki import Wiki +from realms.lib.util import to_canonical, remove_ext, mkdir_safe, gravatar_url +from realms.lib.services import db from models import Site, User, CurrentUser -from ratelimit import get_view_rate_limit, ratelimiter -from services import db + # Flask instance container instances = {} @@ -26,6 +28,17 @@ login_manager = LoginManager() assets = Environment() +class AppCtxGlobals(_AppCtxGlobals): + + @cached_property + def current_user(self): + return session.get('user') if session.get('user') else {'username': 'Anon'} + + +class Application(Flask): + app_ctx_globals_class = AppCtxGlobals + + class SubdomainDispatcher(object): """ Application factory @@ -90,19 +103,12 @@ def redirect_url(referrer=None): return request.args.get('next') or referrer or url_for('index') -def validate_captcha(): - response = captcha.submit( - request.form['recaptcha_challenge_field'], - request.form['recaptcha_response_field'], - config.flask['RECAPTCHA_PRIVATE_KEY'], - request.remote_addr) - return response.is_valid - - def format_subdomain(s): + if not config.repos['enable_subrepos']: + return "" s = s.lower() s = to_canonical(s) - if s in ['www', 'api']: + if s in config.repos['forbidden_subrepos']: # Not allowed s = "" return s @@ -116,7 +122,7 @@ def make_app(subdomain): def create_app(subdomain=None): - app = Flask(__name__) + app = Application(__name__) app.config.update(config.flask) app.debug = (config.ENV is not 'PROD') app.secret_key = config.secret_key @@ -199,7 +205,7 @@ def create_app(subdomain=None): def home(): return redirect(url_for('root')) - @app.route("/account/") + @app.route("/_account/") @login_required def account(): return render_template('account/index.html') @@ -215,13 +221,13 @@ def create_app(subdomain=None): return redirect(redirect_url()) else: s = Site() - s.create(name=wiki_name, repo=wiki_name, founder=CurrentUser.get('id')) + s.create(name=wiki_name, repo=wiki_name, founder=g.current_user.get('id')) instances.pop(wiki_name, None) return redirect('http://%s.%s' % (wiki_name, config.hostname)) else: return render_template('_new/index.html') - @app.route("/logout/") + @app.route("/_logout/") def logout(): User.logout() return redirect(url_for('root')) @@ -234,7 +240,7 @@ def create_app(subdomain=None): if data: return render_template('page/page.html', name=name, page=data, commit=sha) else: - return redirect('/create/'+cname) + return redirect('/_create/'+cname) @app.route("/_compare//") def compare(name, fsha, dots, lsha): @@ -247,11 +253,11 @@ def create_app(subdomain=None): name = request.form.get('name') commit = request.form.get('commit') cname = to_canonical(name) - w.revert_page(name, commit, message="Reverting %s" % cname, username=CurrentUser.get('username')) + w.revert_page(name, commit, message="Reverting %s" % cname, username=g.current_user.get('username')) flash('Page reverted', 'success') return redirect("/" + cname) - @app.route("/register", methods=['GET', 'POST']) + @app.route("/_register", methods=['GET', 'POST']) def register(): if request.method == 'POST': if User.register(request.form.get('username'), request.form.get('email'), request.form.get('password')): @@ -262,14 +268,14 @@ def create_app(subdomain=None): else: return render_template('account/register.html') - @app.route("/login", methods=['GET', 'POST']) + @app.route("/_login", methods=['GET', 'POST']) def login(): if request.method == 'POST': if User.auth(request.form['email'], request.form['password']): return redirect(redirect_url(referrer=url_for('root'))) else: flash("Email or Password invalid") - return redirect("/login") + return redirect("/_login") else: return render_template('account/login.html') @@ -288,7 +294,7 @@ def create_app(subdomain=None): w.rename_page(cname, edit_cname) w.write_page(edit_cname, request.form['content'], message=request.form['message'], - username=CurrentUser.get('username')) + username=g.current_user.get('username')) return redirect("/" + edit_cname) else: if data: @@ -296,7 +302,7 @@ def create_app(subdomain=None): content = data['data'] return render_template('page/edit.html', name=name, content=content) else: - return redirect('/create/'+cname) + return redirect('/_create/'+cname) @app.route("/_delete/", methods=['POST']) @login_required @@ -316,7 +322,7 @@ def create_app(subdomain=None): w.write_page(request.form['name'], request.form['content'], message=request.form['message'], create=True, - username=CurrentUser.get('username')) + username=g.current_user.get('username')) return redirect("/" + cname) else: return render_template('page/edit.html', name=cname, content="") @@ -331,6 +337,6 @@ def create_app(subdomain=None): if data: return render_template('page/page.html', name=cname, page=data) else: - return redirect('/create/'+cname) + return redirect('/_create/'+cname) return app \ No newline at end of file diff --git a/realms/lib/__init__.py b/realms/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/realms/ratelimit.py b/realms/lib/ratelimit.py similarity index 100% rename from realms/ratelimit.py rename to realms/lib/ratelimit.py diff --git a/realms/services.py b/realms/lib/services.py similarity index 85% rename from realms/services.py rename to realms/lib/services.py index bbe6f27..0ad57c4 100644 --- a/realms/services.py +++ b/realms/lib/services.py @@ -1,7 +1,6 @@ import rethinkdb as rdb import redis - -import config +from realms import config # Default DB connection diff --git a/realms/session.py b/realms/lib/session.py similarity index 100% rename from realms/session.py rename to realms/lib/session.py diff --git a/realms/lib/util.py b/realms/lib/util.py new file mode 100644 index 0000000..dd9ee3c --- /dev/null +++ b/realms/lib/util.py @@ -0,0 +1,128 @@ +import re +import os +import hashlib +import json + +from flask import request +from recaptcha.client import captcha + +from realms import config +from realms.lib.services import cache + + +def cache_it(fn): + def wrap(*args, **kw): + key = "%s:%s" % (args[0].table, args[1]) + data = cache.get(key) + # Assume strings are JSON encoded + try: + data = json.loads(data) + except TypeError: + pass + except ValueError: + pass + + if data is not None: + return data + else: + data = fn(*args) + print data + ret = data + if data is None: + data = '' + if not isinstance(data, basestring): + try: + data = json.dumps(data, separators=(',', ':')) + except TypeError: + pass + cache.set(key, data) + return ret + return wrap + + +def to_json(res, first=False): + """ + Jsonify query result. + """ + res = to_dict(res, first) + return json.dumps(res, separators=(',', ':')) + + +def to_dict(cur, first=False): + if not cur: + return None + ret = [] + for row in cur: + ret.append(row) + if ret and first: + return ret[0] + else: + return ret + +def validate_captcha(): + response = captcha.submit( + request.form['recaptcha_challenge_field'], + request.form['recaptcha_response_field'], + config.flask['RECAPTCHA_PRIVATE_KEY'], + request.remote_addr) + return response.is_valid + + +def mkdir_safe(path): + if path and not(os.path.exists(path)): + os.makedirs(path) + return path + + +def extract_path(file_path): + if not file_path: + return None + last_slash = file_path.rindex("/") + if last_slash: + return file_path[0, last_slash] + + +def clean_path(path): + if path: + if path[0] != '/': + path.insert(0, '/') + return re.sub(r"//+", '/', path) + + +def extract_name(file_path): + if file_path[-1] == "/": + return None + return os.path.basename(file_path) + + +def remove_ext(path): + return os.path.splitext(path)[0] + + +def clean_url(url): + if not url: + return url + + url = url.replace('%2F', '/') + url = re.sub(r"^/+", "", url) + return re.sub(r"//+", '/', url) + + +def to_canonical(s): + """ + Double space -> single dash + Double dash -> single dash + Remove all non alphanumeric and dash + Limit to first 64 chars + """ + s = s.encode('ascii', 'ignore') + s = str(s) + s = re.sub(r"\s\s*", "-", s) + s = re.sub(r"\-\-+", "-", s) + s = re.sub(r"[^a-zA-Z0-9\-]", "", s) + s = s[:64] + return s + + +def gravatar_url(email): + return "https://www.gravatar.com/avatar/" + hashlib.md5(email).hexdigest() \ No newline at end of file diff --git a/realms/wiki.py b/realms/lib/wiki.py similarity index 92% rename from realms/wiki.py rename to realms/lib/wiki.py index b31efdf..50353b2 100644 --- a/realms/wiki.py +++ b/realms/lib/wiki.py @@ -7,8 +7,8 @@ import ghdiff import gittle.utils from gittle import Gittle from dulwich.repo import NotGitRepository - -from util import to_canonical, escape_repl, unescape_repl +from werkzeug.utils import escape, unescape +from util import to_canonical from models import Site @@ -80,6 +80,14 @@ class Wiki(): def write_page(self, name, content, message=None, create=False, username=None, email=None): + def escape_repl(m): + if m.group(1): + return "```" + escape(m.group(1)) + "```" + + def unescape_repl(m): + if m.group(1): + return "```" + unescape(m.group(1)) + "```" + # prevents p tag from being added, we remove this later content = '
' + content + '
' content = re.sub(r"```(.*?)```", escape_repl, content, flags=re.DOTALL) @@ -93,9 +101,11 @@ class Wiki(): # post processing to fix errors content = content[5:-6] + # FIXME this is for block quotes, doesn't work for double ">" content = re.sub(r"(\n>)", "\n>", content) content = re.sub(r"(^>)", ">", content) + content = re.sub(r"```(.*?)```", unescape_repl, content, flags=re.DOTALL) filename = self.cname_to_filename(to_canonical(name)) @@ -129,7 +139,7 @@ class Wiki(): files=[old_name]) def get_page(self, name, sha='HEAD'): - commit = gittle.utils.git.commit_info(self.repo[sha]) + # commit = gittle.utils.git.commit_info(self.repo[sha]) name = self.cname_to_filename(name) try: return self.repo.get_commit_files(sha, paths=[name]).get(name) diff --git a/realms/models.py b/realms/models.py index 40bae42..c15808e 100644 --- a/realms/models.py +++ b/realms/models.py @@ -1,60 +1,9 @@ -import json import rethinkdb as rdb import bcrypt from flask import session, flash from flask.ext.login import login_user, logout_user -from util import gravatar_url -from services import db, cache - - -def to_json(res, first=False): - """ - Jsonify query result. - """ - res = to_dict(res, first) - return json.dumps(res, separators=(',',':')) - - -def to_dict(cur, first=False): - if not cur: - return None - ret = [] - for row in cur: - ret.append(row) - if ret and first: - return ret[0] - else: - return ret - - -def cache_it(fn): - def wrap(*args, **kw): - key = "%s:%s" % (args[0].table, args[1]) - data = cache.get(key) - # Assume strings are JSON encoded - try: - data = json.loads(data) - except TypeError: - pass - except ValueError: - pass - - if data is not None: - return data - else: - data = fn(*args) - print data - ret = data - if data is None: - data = '' - if not isinstance(data, basestring): - try: - data = json.dumps(data, separators=(',', ':')) - except TypeError: - pass - cache.set(key, data) - return ret - return wrap +from realms.lib.util import gravatar_url, to_dict, cache_it +from realms.lib.services import db class BaseModel(): @@ -162,7 +111,7 @@ class User(BaseModel): User.login(u.id) @classmethod - def login(cls, id, data=None): + def login(cls, id): login_user(CurrentUser(id), True) @classmethod diff --git a/realms/util.py b/realms/util.py deleted file mode 100644 index 420f74c..0000000 --- a/realms/util.py +++ /dev/null @@ -1,95 +0,0 @@ -import re -import os -import hashlib - - -def escape_repl(m): - print "group 0" - print m.group(0) - print "group 1" - print m.group(1) - if m.group(1): - return "```" + escape_html(m.group(1)) + "```" - - -def unescape_repl(m): - if m.group(1): - return "```" + unescape_html(m.group(1)) + "```" - - -def escape_html(s): - s = s.replace("&", '&') - s = s.replace("<", '<') - s = s.replace(">", '>') - s = s.replace('"', '"') - s = s.replace("'", ''') - return s - - -def unescape_html(s): - s = s.replace('&', "&") - s = s.replace('<', "<") - s = s.replace('>', ">") - s = s.replace('"', '"') - s = s.replace(''', "'") - return s - - -def mkdir_safe(path): - if path and not(os.path.exists(path)): - os.makedirs(path) - return path - - -def extract_path(file_path): - if not file_path: - return None - last_slash = file_path.rindex("/") - if last_slash: - return file_path[0, last_slash] - - -def clean_path(path): - if path: - if path[0] != '/': - path.insert(0, '/') - return re.sub(r"//+", '/', path) - - -def extract_name(file_path): - if file_path[-1] == "/": - return None - return os.path.basename(file_path) - - -def remove_ext(path): - return os.path.splitext(path)[0] - - -def clean_url(url): - if not url: - return url - - url = url.replace('%2F', '/') - url = re.sub(r"^/+", "", url) - return re.sub(r"//+", '/', url) - - -def to_canonical(s): - """ - Double space -> single dash - Double dash -> single dash - Remove all non alphanumeric and dash - Limit to first 64 chars - """ - s = s.encode('ascii', 'ignore') - s = str(s) - s = re.sub(r"\s\s*", "-", s) - s = re.sub(r"\-\-+", "-", s) - s = re.sub(r"[^a-zA-Z0-9\-]", "", s) - s = s[:64] - return s - - -def gravatar_url(email): - return "https://www.gravatar.com/avatar/" + hashlib.md5(email).hexdigest() \ No newline at end of file diff --git a/srv/minion b/srv/minion deleted file mode 100644 index 995d6f7..0000000 --- a/srv/minion +++ /dev/null @@ -1,2 +0,0 @@ -master: localhost -file_client: local diff --git a/srv/salt/common/init.sls b/srv/salt/common/init.sls deleted file mode 100644 index 8340681..0000000 --- a/srv/salt/common/init.sls +++ /dev/null @@ -1,30 +0,0 @@ -extra-repos: - pkgrepo.managed: - - ppa: chris-lea/python-redis - - ppa: brianmercer/redis - - ppa: chris-lea/node.js - - ppa: nginx/stable - -common-pkgs: - pkg.installed: - - pkgs: - - python - - vim - - build-essential - - screen - - htop - - git - - ntp - - libpcre3-dev - - libevent-dev - - iptraf - - python-software-properties - - python-pip - - python-virtualenv - - python-dev - - pkg-config - - curl - - libxml2-dev - - libxslt-dev - - require: - - pkgrepo.managed: extra-repos \ No newline at end of file diff --git a/srv/salt/nginx/init.sls b/srv/salt/nginx/init.sls deleted file mode 100644 index c69ca47..0000000 --- a/srv/salt/nginx/init.sls +++ /dev/null @@ -1,9 +0,0 @@ -nginx: - pkg: - - installed - service: - - running - - enable: True - - reload: True - - require: - - pkg: nginx \ No newline at end of file diff --git a/srv/salt/nginx/nginx.conf b/srv/salt/nginx/nginx.conf deleted file mode 100644 index b6b0931..0000000 --- a/srv/salt/nginx/nginx.conf +++ /dev/null @@ -1,70 +0,0 @@ -{% from 'config.sls' import root %} -gzip_proxied any; -gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; - -upstream web { - fair; - server 127.0.0.1:10000; -} - -server { - listen 80; - - # Allow file uploads - client_max_body_size 50M; - - location ^~ /static/ { - root {{ root }}; - expires max; - } - - location = /favicon.ico { - rewrite (.*) /static/favicon.ico; - } - - location = /robots.txt { - rewrite (.*) /static/robots.txt; - } - - location / { - proxy_pass_header Server; - proxy_redirect off; - proxy_buffering off; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Scheme $scheme; - proxy_set_header Host $http_host; - proxy_pass http://web; - error_page 502 = /maintenance.html; - } - - location /maintenance.html { - root {{ root }}/templates/; - add_header Cache-Control private; - expires epoch; - } -} - -{% if ssl_certificate %} -server { - listen 443; - ssl on; - ssl_certificate {{ ssl_certificate }}; - ssl_certificate_key {{ ssl_certificate_key }}; - - location ^~ /static/ { - root {{ root }}; - expires max; - } - - location / { - proxy_pass_header Server; - proxy_redirect off; - proxy_buffering off; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Scheme $scheme; - proxy_set_header Host $http_host; - proxy_pass http://web; - error_page 502 = /maintenance.html; - } -} -{% endif %} \ No newline at end of file diff --git a/srv/salt/nodejs/init.sls b/srv/salt/nodejs/init.sls deleted file mode 100644 index 487c55f..0000000 --- a/srv/salt/nodejs/init.sls +++ /dev/null @@ -1,13 +0,0 @@ -nodejs: - pkg.installed - -nodejs-dev: - pkg.installed - -npm: - pkg.installed - -bower: - npm.installed: - - require: - - pkg.installed: npm \ No newline at end of file diff --git a/srv/salt/realms/init.sls b/srv/salt/realms/init.sls deleted file mode 100644 index 830472e..0000000 --- a/srv/salt/realms/init.sls +++ /dev/null @@ -1,40 +0,0 @@ -python-dev: - pkg.installed - -python-pip: - pkg.installed - -build-essential: - pkg.installed - -realms-repo: - git.latest: - - unless: test -e /vagrant - - name: git@github.com:scragg0x/realms.git - - target: /home/deploy - - rev: master - - user: deploy - - identity: /home/deploy/.ssh/id_rsa - -realms-link: - cmd.run: - - onlyif: test -e /vagrant - - name: ln -s /vagrant /home/deploy/realms - -/home/deploy/virtualenvs/realms: - file.directory: - - user: deploy - - group: deploy - - makedirs: True - - recurse: - - user - - group - - require: - - user.present: deploy - virtualenv.managed: - - name: /home/deploy/virtualenvs/realms - - requirements: /home/deploy/realms/requirements.txt - - watch: - - git: realms-repo - - require: - - file.directory: /home/deploy/virtualenvs/realms \ No newline at end of file diff --git a/srv/salt/redis/init.sls b/srv/salt/redis/init.sls deleted file mode 100644 index f89b129..0000000 --- a/srv/salt/redis/init.sls +++ /dev/null @@ -1,9 +0,0 @@ -redis-server: - pkg: - - installed - service: - - running - - enable: True - - reload: True - - require: - - pkg: redis-server \ No newline at end of file diff --git a/srv/salt/rethinkdb/init.sls b/srv/salt/rethinkdb/init.sls deleted file mode 100644 index c5ad157..0000000 --- a/srv/salt/rethinkdb/init.sls +++ /dev/null @@ -1,23 +0,0 @@ -rethink-repo: - pkgrepo.managed: - - ppa: rethinkdb/ppa - -rethinkdb: - user.present: - - shell: /bin/bash - - home: /home/rethinkdb - pkg: - - installed - -rethinkdb-pip: - pip: - - name: rethinkdb - - installed - - require: - - pkg: python-pip - - pkg: rethinkdb - - pkg: build-essential - -/etc/rethinkdb/rdb0.conf: - file.managed: - - source: salt://rethinkdb/rdb0.conf diff --git a/srv/salt/rethinkdb/rdb0.conf b/srv/salt/rethinkdb/rdb0.conf deleted file mode 100644 index 26f1ea9..0000000 --- a/srv/salt/rethinkdb/rdb0.conf +++ /dev/null @@ -1,10 +0,0 @@ -runuser=rethinkdb -rungroup=rethinkdb -pid-file=/home/rethinkdb/rdb0/rethinkdb.pid -directory=/home/rethinkdb/rdb0 -bind=all -driver-port=28015 -cluster-port=29015 -port-offset=0 -http-port=20000 -cores=2 \ No newline at end of file diff --git a/srv/salt/supervisor/init.sls b/srv/salt/supervisor/init.sls deleted file mode 100644 index 4682722..0000000 --- a/srv/salt/supervisor/init.sls +++ /dev/null @@ -1,19 +0,0 @@ -/etc/supervisord.conf: - file.managed: - - source: salt://supervisor/supervisord.conf - - -supervisor-pip: - pip: - - name: supervisor - - installed - - require: - - pkg.installed: python-pip - -supervisor-run: - cmd.run: - - unless: test -e /tmp/supervisord.pid - - name: /usr/local/bin/supervisord - - require: - - file.managed: /etc/supervisord.conf - - file.managed: /etc/rethinkdb/rdb0.conf \ No newline at end of file diff --git a/srv/salt/supervisor/supervisord.conf b/srv/salt/supervisor/supervisord.conf deleted file mode 100644 index 4035000..0000000 --- a/srv/salt/supervisor/supervisord.conf +++ /dev/null @@ -1,33 +0,0 @@ -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[unix_http_server] -file = /tmp/supervisor.sock -chmod = 0777 - -[supervisorctl] -serverurl = unix:///tmp/supervisor.sock - -[supervisord] -logfile = /tmp/supervisord.log -logfile_maxbytes = 50MB -logfile_backups=10 -loglevel = info -pidfile = /tmp/supervisord.pid -nodaemon = false -minfds = 1024 -minprocs = 200 -umask = 022 -user = root -identifier = supervisor -directory = /tmp -nocleanup = true -childlogdir = /tmp -strip_ansi = false - -[program:realms] -command=/home/deploy/virtualenvs/realms/bin/python /home/deploy/realms/app.py - -[program:rethinkdb] -command=/usr/bin/rethinkdb --config-file /etc/rethinkdb/rdb0.conf -user=root \ No newline at end of file diff --git a/srv/salt/top.sls b/srv/salt/top.sls deleted file mode 100644 index e051b8f..0000000 --- a/srv/salt/top.sls +++ /dev/null @@ -1,10 +0,0 @@ -base: - '*': - - common - - users - - nodejs - - redis - - nginx - - rethinkdb - - realms - - supervisor \ No newline at end of file diff --git a/srv/salt/users/init.sls b/srv/salt/users/init.sls deleted file mode 100644 index 6c0750f..0000000 --- a/srv/salt/users/init.sls +++ /dev/null @@ -1,19 +0,0 @@ -deploy: - user.present: - - shell: /bin/bash - - home: /home/deploy - - fullname: Deploy - -sudo: - pkg: - - installed - -/etc/sudoes.d/mysudoers: - file: - - managed - - source: salt://users/mysudoers - - mode: 440 - - user: root - - group: root - - require: - - pkg: sudo diff --git a/srv/salt/users/mysudoers b/srv/salt/users/mysudoers deleted file mode 100644 index 8790061..0000000 --- a/srv/salt/users/mysudoers +++ /dev/null @@ -1 +0,0 @@ -deploy ALL=(ALL) NOPASSWD:ALL \ No newline at end of file