reorg

Vagrantfile

@@ -1,16 +0,0 @@
-VAGRANTFILE_API_VERSION = "2"
-
-Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.box = "precise64"
-  config.vm.synced_folder "srv/", "/srv/"
-  config.vm.provision :salt do |salt|
-  	salt.minion_config = "srv/minion"
-	salt.run_highstate = true
-  end
-end
-
-Vagrant::Config.run do |config|
-  config.vm.forward_port 80, 8000
-  config.vm.forward_port 10000, 10000
-  config.vm.forward_port 20000, 20000
-end

realms/__init__.py

@@ -4,19 +4,21 @@ import time
 from threading import Lock
 
 import rethinkdb as rdb
-from flask import Flask, request, render_template, url_for, redirect, flash
+from flask import Flask, g, request, render_template, url_for, redirect, flash, session
+from flask.ctx import _AppCtxGlobals
 from flask.ext.login import LoginManager, login_required
 from flask.ext.assets import Environment, Bundle
-from recaptcha.client import captcha
 from werkzeug.routing import BaseConverter
+from werkzeug.utils import cached_property
 
-from session import RedisSessionInterface
 import config
-from wiki import Wiki
+from realms.lib.ratelimit import get_view_rate_limit, ratelimiter
-from util import to_canonical, remove_ext, mkdir_safe, gravatar_url
+from realms.lib.session import RedisSessionInterface
+from realms.lib.wiki import Wiki
+from realms.lib.util import to_canonical, remove_ext, mkdir_safe, gravatar_url
+from realms.lib.services import db
 from models import Site, User, CurrentUser
-from ratelimit import get_view_rate_limit, ratelimiter
+
-from services import db
 
 # Flask instance container
 instances = {}
@@ -26,6 +28,17 @@ login_manager = LoginManager()
 assets = Environment()
 
 
+class AppCtxGlobals(_AppCtxGlobals):
+
+    @cached_property
+    def current_user(self):
+        return session.get('user') if session.get('user') else {'username': 'Anon'}
+
+
+class Application(Flask):
+    app_ctx_globals_class = AppCtxGlobals
+
+
 class SubdomainDispatcher(object):
     """
     Application factory
@@ -90,19 +103,12 @@ def redirect_url(referrer=None):
     return request.args.get('next') or referrer or url_for('index')
 
 
-def validate_captcha():
-    response = captcha.submit(
-        request.form['recaptcha_challenge_field'],
-        request.form['recaptcha_response_field'],
-        config.flask['RECAPTCHA_PRIVATE_KEY'],
-        request.remote_addr)
-    return response.is_valid
-
-
 def format_subdomain(s):
+    if not config.repos['enable_subrepos']:
+        return ""
     s = s.lower()
     s = to_canonical(s)
-    if s in ['www', 'api']:
+    if s in config.repos['forbidden_subrepos']:
         # Not allowed
         s = ""
     return s
@@ -116,7 +122,7 @@ def make_app(subdomain):
 
 
 def create_app(subdomain=None):
-    app = Flask(__name__)
+    app = Application(__name__)
     app.config.update(config.flask)
     app.debug = (config.ENV is not 'PROD')
     app.secret_key = config.secret_key
@@ -199,7 +205,7 @@ def create_app(subdomain=None):
     def home():
         return redirect(url_for('root'))
 
-    @app.route("/account/")
+    @app.route("/_account/")
     @login_required
     def account():
         return render_template('account/index.html')
@@ -215,13 +221,13 @@ def create_app(subdomain=None):
                 return redirect(redirect_url())
             else:
                 s = Site()
-                s.create(name=wiki_name, repo=wiki_name, founder=CurrentUser.get('id'))
+                s.create(name=wiki_name, repo=wiki_name, founder=g.current_user.get('id'))
                 instances.pop(wiki_name, None)
                 return redirect('http://%s.%s' % (wiki_name, config.hostname))
         else:
             return render_template('_new/index.html')
 
-    @app.route("/logout/")
+    @app.route("/_logout/")
     def logout():
         User.logout()
         return redirect(url_for('root'))
@@ -234,7 +240,7 @@ def create_app(subdomain=None):
         if data:
             return render_template('page/page.html', name=name, page=data, commit=sha)
         else:
-            return redirect('/create/'+cname)
+            return redirect('/_create/'+cname)
 
     @app.route("/_compare/<name>/<regex('[^.]+'):fsha><regex('\.{2,3}'):dots><regex('.+'):lsha>")
     def compare(name, fsha, dots, lsha):
@@ -247,11 +253,11 @@ def create_app(subdomain=None):
             name = request.form.get('name')
             commit = request.form.get('commit')
             cname = to_canonical(name)
-            w.revert_page(name, commit, message="Reverting %s" % cname, username=CurrentUser.get('username'))
+            w.revert_page(name, commit, message="Reverting %s" % cname, username=g.current_user.get('username'))
             flash('Page reverted', 'success')
             return redirect("/" + cname)
 
-    @app.route("/register", methods=['GET', 'POST'])
+    @app.route("/_register", methods=['GET', 'POST'])
     def register():
         if request.method == 'POST':
             if User.register(request.form.get('username'), request.form.get('email'), request.form.get('password')):
@@ -262,14 +268,14 @@ def create_app(subdomain=None):
         else:
             return render_template('account/register.html')
 
-    @app.route("/login", methods=['GET', 'POST'])
+    @app.route("/_login", methods=['GET', 'POST'])
     def login():
         if request.method == 'POST':
             if User.auth(request.form['email'], request.form['password']):
                 return redirect(redirect_url(referrer=url_for('root')))
             else:
                 flash("Email or Password invalid")
-                return redirect("/login")
+                return redirect("/_login")
         else:
             return render_template('account/login.html')
 
@@ -288,7 +294,7 @@ def create_app(subdomain=None):
                 w.rename_page(cname, edit_cname)
             w.write_page(edit_cname, request.form['content'],
                          message=request.form['message'],
-                         username=CurrentUser.get('username'))
+                         username=g.current_user.get('username'))
             return redirect("/" + edit_cname)
         else:
             if data:
@@ -296,7 +302,7 @@ def create_app(subdomain=None):
                 content = data['data']
                 return render_template('page/edit.html', name=name, content=content)
             else:
-                return redirect('/create/'+cname)
+                return redirect('/_create/'+cname)
 
     @app.route("/_delete/<name>", methods=['POST'])
     @login_required
@@ -316,7 +322,7 @@ def create_app(subdomain=None):
             w.write_page(request.form['name'], request.form['content'],
                          message=request.form['message'],
                          create=True,
-                         username=CurrentUser.get('username'))
+                         username=g.current_user.get('username'))
             return redirect("/" + cname)
         else:
             return render_template('page/edit.html', name=cname, content="")
@@ -331,6 +337,6 @@ def create_app(subdomain=None):
         if data:
             return render_template('page/page.html', name=cname, page=data)
         else:
-            return redirect('/create/'+cname)
+            return redirect('/_create/'+cname)
 
     return app

realms/lib/services.py

@@ -1,7 +1,6 @@
 import rethinkdb as rdb
 import redis
-
+from realms import config
-import config
 
 
 # Default DB connection

realms/lib/util.py

@@ -0,0 +1,128 @@
+import re
+import os
+import hashlib
+import json
+
+from flask import request
+from recaptcha.client import captcha
+
+from realms import config
+from realms.lib.services import cache
+
+
+def cache_it(fn):
+    def wrap(*args, **kw):
+        key = "%s:%s" % (args[0].table, args[1])
+        data = cache.get(key)
+        # Assume strings are JSON encoded
+        try:
+            data = json.loads(data)
+        except TypeError:
+            pass
+        except ValueError:
+            pass
+
+        if data is not None:
+            return data
+        else:
+            data = fn(*args)
+            print data
+            ret = data
+            if data is None:
+                data = ''
+            if not isinstance(data, basestring):
+                try:
+                    data = json.dumps(data, separators=(',', ':'))
+                except TypeError:
+                    pass
+            cache.set(key, data)
+            return ret
+    return wrap
+
+
+def to_json(res, first=False):
+    """
+    Jsonify query result.
+    """
+    res = to_dict(res, first)
+    return json.dumps(res, separators=(',', ':'))
+
+
+def to_dict(cur, first=False):
+    if not cur:
+        return None
+    ret = []
+    for row in cur:
+        ret.append(row)
+    if ret and first:
+        return ret[0]
+    else:
+        return ret
+
+def validate_captcha():
+    response = captcha.submit(
+        request.form['recaptcha_challenge_field'],
+        request.form['recaptcha_response_field'],
+        config.flask['RECAPTCHA_PRIVATE_KEY'],
+        request.remote_addr)
+    return response.is_valid
+
+
+def mkdir_safe(path):
+    if path and not(os.path.exists(path)):
+        os.makedirs(path)
+    return path
+
+
+def extract_path(file_path):
+    if not file_path:
+        return None
+    last_slash = file_path.rindex("/")
+    if last_slash:
+        return file_path[0, last_slash]
+
+
+def clean_path(path):
+    if path:
+        if path[0] != '/':
+            path.insert(0, '/')
+        return re.sub(r"//+", '/', path)
+
+
+def extract_name(file_path):
+    if file_path[-1] == "/":
+        return None
+    return os.path.basename(file_path)
+
+
+def remove_ext(path):
+    return os.path.splitext(path)[0]
+
+
+def clean_url(url):
+    if not url:
+        return url
+
+    url = url.replace('%2F', '/')
+    url = re.sub(r"^/+", "", url)
+    return re.sub(r"//+", '/', url)
+
+
+def to_canonical(s):
+    """
+    Double space -> single dash
+    Double dash -> single dash
+    Remove all non alphanumeric and dash
+    Limit to first 64 chars
+    """
+    s = s.encode('ascii', 'ignore')
+    s = str(s)
+    s = re.sub(r"\s\s*", "-", s)
+    s = re.sub(r"\-\-+", "-", s)
+    s = re.sub(r"[^a-zA-Z0-9\-]", "", s)
+    s = s[:64]
+    return s
+
+
+def gravatar_url(email):
+    return "https://www.gravatar.com/avatar/" + hashlib.md5(email).hexdigest()

realms/lib/wiki.py

@@ -7,8 +7,8 @@ import ghdiff
 import gittle.utils
 from gittle import Gittle
 from dulwich.repo import NotGitRepository
-
+from werkzeug.utils import escape, unescape
-from util import to_canonical, escape_repl, unescape_repl
+from util import to_canonical
 from models import Site
 
 
@@ -80,6 +80,14 @@ class Wiki():
 
     def write_page(self, name, content, message=None, create=False, username=None, email=None):
 
+        def escape_repl(m):
+            if m.group(1):
+                return "```" + escape(m.group(1)) + "```"
+
+        def unescape_repl(m):
+            if m.group(1):
+                return "```" + unescape(m.group(1)) + "```"
+
         # prevents p tag from being added, we remove this later
         content = '<div>' + content + '</div>'
         content = re.sub(r"```(.*?)```", escape_repl, content, flags=re.DOTALL)
@@ -93,9 +101,11 @@ class Wiki():
 
         # post processing to fix errors
         content = content[5:-6]
+
         # FIXME this is for block quotes, doesn't work for double ">"
         content = re.sub(r"(\n&gt;)", "\n>", content)
         content = re.sub(r"(^&gt;)", ">", content)
+
         content = re.sub(r"```(.*?)```", unescape_repl, content, flags=re.DOTALL)
 
         filename = self.cname_to_filename(to_canonical(name))
@@ -129,7 +139,7 @@ class Wiki():
                          files=[old_name])
 
     def get_page(self, name, sha='HEAD'):
-        commit = gittle.utils.git.commit_info(self.repo[sha])
+        # commit = gittle.utils.git.commit_info(self.repo[sha])
         name = self.cname_to_filename(name)
         try:
             return self.repo.get_commit_files(sha, paths=[name]).get(name)

realms/models.py

@@ -1,60 +1,9 @@
-import json
 import rethinkdb as rdb
 import bcrypt
 from flask import session, flash
 from flask.ext.login import login_user, logout_user
-from util import gravatar_url
+from realms.lib.util import gravatar_url, to_dict, cache_it
-from services import db, cache
+from realms.lib.services import db
-
-
-def to_json(res, first=False):
-    """
-    Jsonify query result.
-    """
-    res = to_dict(res, first)
-    return json.dumps(res, separators=(',',':'))
-
-
-def to_dict(cur, first=False):
-    if not cur:
-        return None
-    ret = []
-    for row in cur:
-        ret.append(row)
-    if ret and first:
-        return ret[0]
-    else:
-        return ret
-
-
-def cache_it(fn):
-    def wrap(*args, **kw):
-        key = "%s:%s" % (args[0].table, args[1])
-        data = cache.get(key)
-        # Assume strings are JSON encoded
-        try:
-            data = json.loads(data)
-        except TypeError:
-            pass
-        except ValueError:
-            pass
-
-        if data is not None:
-            return data
-        else:
-            data = fn(*args)
-            print data
-            ret = data
-            if data is None:
-                data = ''
-            if not isinstance(data, basestring):
-                try:
-                    data = json.dumps(data, separators=(',', ':'))
-                except TypeError:
-                    pass
-            cache.set(key, data)
-            return ret
-    return wrap
 
 
 class BaseModel():
@@ -162,7 +111,7 @@ class User(BaseModel):
         User.login(u.id)
 
     @classmethod
-    def login(cls, id, data=None):
+    def login(cls, id):
         login_user(CurrentUser(id), True)
 
     @classmethod

realms/util.py

@@ -1,95 +0,0 @@
-import re
-import os
-import hashlib
-
-
-def escape_repl(m):
-    print "group 0"
-    print m.group(0)
-    print "group 1"
-    print m.group(1)
-    if m.group(1):
-        return "```" + escape_html(m.group(1)) + "```"
-
-
-def unescape_repl(m):
-    if m.group(1):
-        return "```" + unescape_html(m.group(1)) + "```"
-
-
-def escape_html(s):
-    s = s.replace("&", '&')
-    s = s.replace("<", '&lt;')
-    s = s.replace(">", '&gt;')
-    s = s.replace('"', '&quot;')
-    s = s.replace("'", '&#39;')
-    return s
-
-
-def unescape_html(s):
-    s = s.replace('&amp;', "&")
-    s = s.replace('&lt;', "<")
-    s = s.replace('&gt;', ">")
-    s = s.replace('&quot;', '"')
-    s = s.replace('&#39;', "'")
-    return s
-
-
-def mkdir_safe(path):
-    if path and not(os.path.exists(path)):
-        os.makedirs(path)
-    return path
-
-
-def extract_path(file_path):
-    if not file_path:
-        return None
-    last_slash = file_path.rindex("/")
-    if last_slash:
-        return file_path[0, last_slash]
-
-
-def clean_path(path):
-    if path:
-        if path[0] != '/':
-            path.insert(0, '/')
-        return re.sub(r"//+", '/', path)
-
-
-def extract_name(file_path):
-    if file_path[-1] == "/":
-        return None
-    return os.path.basename(file_path)
-
-
-def remove_ext(path):
-    return os.path.splitext(path)[0]
-
-
-def clean_url(url):
-    if not url:
-        return url
-
-    url = url.replace('%2F', '/')
-    url = re.sub(r"^/+", "", url)
-    return re.sub(r"//+", '/', url)
-
-
-def to_canonical(s):
-    """
-    Double space -> single dash
-    Double dash -> single dash
-    Remove all non alphanumeric and dash
-    Limit to first 64 chars
-    """
-    s = s.encode('ascii', 'ignore')
-    s = str(s)
-    s = re.sub(r"\s\s*", "-", s)
-    s = re.sub(r"\-\-+", "-", s)
-    s = re.sub(r"[^a-zA-Z0-9\-]", "", s)
-    s = s[:64]
-    return s
-
-
-def gravatar_url(email):
-    return "https://www.gravatar.com/avatar/" + hashlib.md5(email).hexdigest()

srv/minion

@@ -1,2 +0,0 @@
-master: localhost
-file_client: local

srv/salt/common/init.sls

@@ -1,30 +0,0 @@
-extra-repos:
-  pkgrepo.managed:
-    - ppa: chris-lea/python-redis
-    - ppa: brianmercer/redis
-    - ppa: chris-lea/node.js
-    - ppa: nginx/stable
-
-common-pkgs:
-  pkg.installed:
-    - pkgs:
-      - python
-      - vim
-      - build-essential
-      - screen
-      - htop
-      - git
-      - ntp
-      - libpcre3-dev
-      - libevent-dev
-      - iptraf
-      - python-software-properties
-      - python-pip
-      - python-virtualenv
-      - python-dev
-      - pkg-config
-      - curl
-      - libxml2-dev
-      - libxslt-dev
-    - require:
-      - pkgrepo.managed: extra-repos

srv/salt/nginx/init.sls

@@ -1,9 +0,0 @@
-nginx:
-  pkg:
-    - installed
-  service:
-    - running
-    - enable: True
-    - reload: True
-    - require:
-      - pkg: nginx

srv/salt/nginx/nginx.conf

@@ -1,70 +0,0 @@
-{% from 'config.sls' import root %}
-gzip_proxied any;
-gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
-
-upstream web {
-  fair;
-  server 127.0.0.1:10000;
-}
-
-server {
-  listen 80;
-
-  # Allow file uploads
-  client_max_body_size 50M;
-
-  location ^~ /static/ {
-    root {{ root }};
-    expires max;
-  }
-
-  location = /favicon.ico {
-    rewrite (.*) /static/favicon.ico;
-  }
-
-  location = /robots.txt {
-    rewrite (.*) /static/robots.txt;
-  }
-
-  location / {
-      proxy_pass_header Server;
-      proxy_redirect off;
-      proxy_buffering off;
-      proxy_set_header X-Real-IP $remote_addr;
-      proxy_set_header X-Scheme $scheme;
-      proxy_set_header Host $http_host;
-      proxy_pass http://web;
-    error_page 502 = /maintenance.html;
-  }
-
-  location /maintenance.html {
-    root {{ root }}/templates/;
-    add_header Cache-Control private;
-    expires epoch;
-  }
-}
-
-{% if ssl_certificate %}
-server {
-  listen 443;
-  ssl on;
-  ssl_certificate {{ ssl_certificate }};
-  ssl_certificate_key {{ ssl_certificate_key }};
-
-  location ^~ /static/ {
-    root {{ root }};
-    expires max;
-  }
-
-  location / {
-    proxy_pass_header Server;
-    proxy_redirect off;
-    proxy_buffering off;
-    proxy_set_header X-Real-IP $remote_addr;
-    proxy_set_header X-Scheme $scheme;
-    proxy_set_header Host $http_host;
-    proxy_pass http://web;
-    error_page 502 = /maintenance.html;
-  }
-}
-{% endif %}

srv/salt/nodejs/init.sls

@@ -1,13 +0,0 @@
-nodejs:
-  pkg.installed
-
-nodejs-dev:
-  pkg.installed
-
-npm:
-  pkg.installed
-
-bower:
-  npm.installed:
-    - require:
-      - pkg.installed: npm

srv/salt/realms/init.sls

@@ -1,40 +0,0 @@
-python-dev:
-  pkg.installed
-
-python-pip:
-  pkg.installed
-
-build-essential:
-  pkg.installed
-
-realms-repo:
-  git.latest:
-    - unless: test -e /vagrant
-    - name: git@github.com:scragg0x/realms.git
-    - target: /home/deploy
-    - rev: master
-    - user: deploy
-    - identity: /home/deploy/.ssh/id_rsa
-
-realms-link:
-  cmd.run:
-    - onlyif: test -e /vagrant
-    - name: ln -s /vagrant /home/deploy/realms
-
-/home/deploy/virtualenvs/realms:
-  file.directory:
-    - user: deploy
-    - group: deploy
-    - makedirs: True
-    - recurse:
-      - user
-      - group
-    - require:
-      - user.present: deploy
-  virtualenv.managed:
-    - name: /home/deploy/virtualenvs/realms
-    - requirements: /home/deploy/realms/requirements.txt
-    - watch:
-      - git: realms-repo
-    - require:
-      - file.directory: /home/deploy/virtualenvs/realms

srv/salt/redis/init.sls

@@ -1,9 +0,0 @@
-redis-server:
-  pkg:
-    - installed
-  service:
-    - running
-    - enable: True
-    - reload: True
-    - require:
-      - pkg: redis-server

srv/salt/rethinkdb/init.sls

@@ -1,23 +0,0 @@
-rethink-repo:
-  pkgrepo.managed:
-    - ppa: rethinkdb/ppa
-
-rethinkdb:
-  user.present:
-    - shell: /bin/bash
-    - home: /home/rethinkdb
-  pkg:
-    - installed
-
-rethinkdb-pip:
-  pip:
-    - name: rethinkdb
-    - installed
-    - require:
-      - pkg: python-pip
-      - pkg: rethinkdb
-      - pkg: build-essential
-
-/etc/rethinkdb/rdb0.conf:
-  file.managed:
-    - source: salt://rethinkdb/rdb0.conf

srv/salt/rethinkdb/rdb0.conf

@@ -1,10 +0,0 @@
-runuser=rethinkdb
-rungroup=rethinkdb
-pid-file=/home/rethinkdb/rdb0/rethinkdb.pid
-directory=/home/rethinkdb/rdb0
-bind=all
-driver-port=28015
-cluster-port=29015
-port-offset=0
-http-port=20000
-cores=2

srv/salt/supervisor/init.sls

@@ -1,19 +0,0 @@
-/etc/supervisord.conf:
-  file.managed:
-    - source: salt://supervisor/supervisord.conf
-
-
-supervisor-pip:
-  pip:
-    - name: supervisor
-    - installed
-    - require:
-      - pkg.installed: python-pip
-
-supervisor-run:
-  cmd.run:
-    - unless: test -e /tmp/supervisord.pid
-    - name: /usr/local/bin/supervisord
-    - require:
-      - file.managed: /etc/supervisord.conf
-      - file.managed: /etc/rethinkdb/rdb0.conf

srv/salt/supervisor/supervisord.conf

@@ -1,33 +0,0 @@
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[unix_http_server]
-file = /tmp/supervisor.sock
-chmod = 0777
-
-[supervisorctl]
-serverurl = unix:///tmp/supervisor.sock
-
-[supervisord]
-logfile = /tmp/supervisord.log
-logfile_maxbytes = 50MB
-logfile_backups=10
-loglevel = info
-pidfile = /tmp/supervisord.pid
-nodaemon = false
-minfds = 1024
-minprocs = 200
-umask = 022
-user = root
-identifier = supervisor
-directory = /tmp
-nocleanup = true
-childlogdir = /tmp
-strip_ansi = false
-
-[program:realms]
-command=/home/deploy/virtualenvs/realms/bin/python /home/deploy/realms/app.py
-
-[program:rethinkdb]
-command=/usr/bin/rethinkdb --config-file /etc/rethinkdb/rdb0.conf
-user=root

srv/salt/top.sls

@@ -1,10 +0,0 @@
-base:
-  '*':
-    - common
-    - users
-    - nodejs
-    - redis
-    - nginx
-    - rethinkdb
-    - realms
-    - supervisor

srv/salt/users/init.sls

@@ -1,19 +0,0 @@
-deploy:
-  user.present:
-    - shell: /bin/bash
-    - home: /home/deploy
-    - fullname: Deploy
-
-sudo:
-  pkg:
-    - installed
-
-/etc/sudoes.d/mysudoers:
-  file:
-    - managed
-    - source: salt://users/mysudoers
-    - mode: 440
-    - user: root
-    - group: root
-    - require:
-      - pkg: sudo

srv/salt/users/mysudoers

@@ -1 +0,0 @@
-deploy ALL=(ALL) NOPASSWD:ALL