reorg

realms/lib/ratelimit.py

@@ -0,0 +1,46 @@
+import time
+from functools import update_wrapper
+from flask import request, g
+from services import cache
+
+
+class RateLimit(object):
+    expiration_window = 10
+
+    def __init__(self, key_prefix, limit, per, send_x_headers):
+        self.reset = (int(time.time()) // per) * per + per
+        self.key = key_prefix + str(self.reset)
+        self.limit = limit
+        self.per = per
+        self.send_x_headers = send_x_headers
+        p = cache.pipeline()
+        p.incr(self.key)
+        p.expireat(self.key, self.reset + self.expiration_window)
+        self.current = min(p.execute()[0], limit)
+
+    remaining = property(lambda x: x.limit - x.current)
+    over_limit = property(lambda x: x.current >= x.limit)
+
+
+def get_view_rate_limit():
+    return getattr(g, '_view_rate_limit', None)
+
+
+def on_over_limit(limit):
+    return 'Slow it down', 400
+
+
+def ratelimiter(limit, per=300, send_x_headers=True,
+              over_limit=on_over_limit,
+              scope_func=lambda: request.remote_addr,
+              key_func=lambda: request.endpoint):
+    def decorator(f):
+        def rate_limited(*args, **kwargs):
+            key = 'rate-limit/%s/%s/' % (key_func(), scope_func())
+            rlimit = RateLimit(key, limit, per, send_x_headers)
+            g._view_rate_limit = rlimit
+            if over_limit is not None and rlimit.over_limit:
+                return over_limit(rlimit)
+            return f(*args, **kwargs)
+        return update_wrapper(rate_limited, f)
+    return decorator

realms/lib/services.py

@@ -0,0 +1,10 @@
+import rethinkdb as rdb
+import redis
+from realms import config
+
+
+# Default DB connection
+db = rdb.connect(config.db['host'], config.db['port'], db=config.db['dbname'])
+
+# Default Cache connection
+cache = redis.StrictRedis(host=config.cache['host'], port=config.cache['port'])

realms/lib/session.py

@@ -0,0 +1,64 @@
+import pickle
+from datetime import timedelta
+from uuid import uuid4
+from redis import Redis
+from werkzeug.datastructures import CallbackDict
+from flask.sessions import SessionInterface, SessionMixin
+
+
+class RedisSession(CallbackDict, SessionMixin):
+
+    def __init__(self, initial=None, sid=None, new=False):
+        def on_update(self):
+            self.modified = True
+        CallbackDict.__init__(self, initial, on_update)
+        self.sid = sid
+        self.new = new
+        self.modified = False
+
+
+class RedisSessionInterface(SessionInterface):
+    serializer = pickle
+    session_class = RedisSession
+
+    def __init__(self, redis=None, prefix='session:'):
+        if redis is None:
+            redis = Redis()
+        self.redis = redis
+        self.prefix = prefix
+
+    def generate_sid(self):
+        return str(uuid4())
+
+    def get_redis_expiration_time(self, app, session):
+        if session.permanent:
+            return app.permanent_session_lifetime
+        return timedelta(days=1)
+
+    def open_session(self, app, request):
+        sid = request.cookies.get(app.session_cookie_name)
+        if not sid:
+            sid = self.generate_sid()
+            return self.session_class(sid=sid)
+        val = self.redis.get(self.prefix + sid)
+        if val is not None:
+            data = self.serializer.loads(val)
+            return self.session_class(data, sid=sid)
+        return self.session_class(sid=sid, new=True)
+
+    def save_session(self, app, session, response):
+        domain = self.get_cookie_domain(app)
+        if not session:
+            self.redis.delete(self.prefix + session.sid)
+            if session.modified:
+                response.delete_cookie(app.session_cookie_name,
+                                       domain=domain)
+            return
+        redis_exp = self.get_redis_expiration_time(app, session)
+        cookie_exp = self.get_expiration_time(app, session)
+        val = self.serializer.dumps(dict(session))
+        self.redis.setex(self.prefix + session.sid, val,
+                         int(redis_exp.total_seconds()))
+        response.set_cookie(app.session_cookie_name, session.sid,
+                            expires=cookie_exp, httponly=True,
+                            domain=domain)

realms/lib/util.py

@@ -0,0 +1,128 @@
+import re
+import os
+import hashlib
+import json
+
+from flask import request
+from recaptcha.client import captcha
+
+from realms import config
+from realms.lib.services import cache
+
+
+def cache_it(fn):
+    def wrap(*args, **kw):
+        key = "%s:%s" % (args[0].table, args[1])
+        data = cache.get(key)
+        # Assume strings are JSON encoded
+        try:
+            data = json.loads(data)
+        except TypeError:
+            pass
+        except ValueError:
+            pass
+
+        if data is not None:
+            return data
+        else:
+            data = fn(*args)
+            print data
+            ret = data
+            if data is None:
+                data = ''
+            if not isinstance(data, basestring):
+                try:
+                    data = json.dumps(data, separators=(',', ':'))
+                except TypeError:
+                    pass
+            cache.set(key, data)
+            return ret
+    return wrap
+
+
+def to_json(res, first=False):
+    """
+    Jsonify query result.
+    """
+    res = to_dict(res, first)
+    return json.dumps(res, separators=(',', ':'))
+
+
+def to_dict(cur, first=False):
+    if not cur:
+        return None
+    ret = []
+    for row in cur:
+        ret.append(row)
+    if ret and first:
+        return ret[0]
+    else:
+        return ret
+
+def validate_captcha():
+    response = captcha.submit(
+        request.form['recaptcha_challenge_field'],
+        request.form['recaptcha_response_field'],
+        config.flask['RECAPTCHA_PRIVATE_KEY'],
+        request.remote_addr)
+    return response.is_valid
+
+
+def mkdir_safe(path):
+    if path and not(os.path.exists(path)):
+        os.makedirs(path)
+    return path
+
+
+def extract_path(file_path):
+    if not file_path:
+        return None
+    last_slash = file_path.rindex("/")
+    if last_slash:
+        return file_path[0, last_slash]
+
+
+def clean_path(path):
+    if path:
+        if path[0] != '/':
+            path.insert(0, '/')
+        return re.sub(r"//+", '/', path)
+
+
+def extract_name(file_path):
+    if file_path[-1] == "/":
+        return None
+    return os.path.basename(file_path)
+
+
+def remove_ext(path):
+    return os.path.splitext(path)[0]
+
+
+def clean_url(url):
+    if not url:
+        return url
+
+    url = url.replace('%2F', '/')
+    url = re.sub(r"^/+", "", url)
+    return re.sub(r"//+", '/', url)
+
+
+def to_canonical(s):
+    """
+    Double space -> single dash
+    Double dash -> single dash
+    Remove all non alphanumeric and dash
+    Limit to first 64 chars
+    """
+    s = s.encode('ascii', 'ignore')
+    s = str(s)
+    s = re.sub(r"\s\s*", "-", s)
+    s = re.sub(r"\-\-+", "-", s)
+    s = re.sub(r"[^a-zA-Z0-9\-]", "", s)
+    s = s[:64]
+    return s
+
+
+def gravatar_url(email):
+    return "https://www.gravatar.com/avatar/" + hashlib.md5(email).hexdigest()

realms/lib/wiki.py

@@ -0,0 +1,159 @@
+import os
+import re
+import lxml.html
+from lxml.html import clean
+import ghdiff
+
+import gittle.utils
+from gittle import Gittle
+from dulwich.repo import NotGitRepository
+from werkzeug.utils import escape, unescape
+from util import to_canonical
+from models import Site
+
+
+class MyGittle(Gittle):
+
+    def file_history(self, path):
+        """Returns all commits where given file was modified
+        """
+        versions = []
+        commits_info = self.commit_info()
+        seen_shas = set()
+
+        for commit in commits_info:
+            try:
+                files = self.get_commit_files(commit['sha'], paths=[path])
+                file_path, file_data = files.items()[0]
+            except IndexError:
+                continue
+
+            file_sha = file_data['sha']
+
+            if file_sha in seen_shas:
+                continue
+            else:
+                seen_shas.add(file_sha)
+
+            versions.append(dict(author=commit['author']['name'],
+                                 time=commit['time'],
+                                 file_sha=file_sha,
+                                 sha=commit['sha'],
+                                 message=commit['message']))
+        return versions
+
+    def mv_fs(self, file_pair):
+        old_name, new_name = file_pair
+        os.rename(self.path + "/" + old_name, self.path + "/" + new_name)
+
+
+class Wiki():
+    path = None
+    base_path = '/'
+    default_ref = 'master'
+    default_committer_name = 'Anon'
+    default_committer_email = 'anon@anon.anon'
+    index_page = 'home'
+    repo = None
+
+    def __init__(self, path):
+        try:
+            self.repo = MyGittle(path)
+        except NotGitRepository:
+            self.repo = MyGittle.init(path)
+
+        self.path = path
+
+    @staticmethod
+    def is_registered(name):
+        s = Site()
+        return True if s.get_by_name(name) else False
+
+    def revert_page(self, name, commit_sha, message, username):
+        page = self.get_page(name, commit_sha)
+        if not page:
+            # Page not found
+            return None
+        commit_info = gittle.utils.git.commit_info(self.repo[commit_sha])
+        message = commit_info['message']
+        return self.write_page(name, page['data'], message=message, username=username)
+
+    def write_page(self, name, content, message=None, create=False, username=None, email=None):
+
+        def escape_repl(m):
+            if m.group(1):
+                return "```" + escape(m.group(1)) + "```"
+
+        def unescape_repl(m):
+            if m.group(1):
+                return "```" + unescape(m.group(1)) + "```"
+
+        # prevents p tag from being added, we remove this later
+        content = '<div>' + content + '</div>'
+        content = re.sub(r"```(.*?)```", escape_repl, content, flags=re.DOTALL)
+
+        tree = lxml.html.fromstring(content)
+
+        cleaner = clean.Cleaner(remove_unknown_tags=False, kill_tags=set(['style']), safe_attrs_only=False)
+        tree = cleaner.clean_html(tree)
+
+        content = lxml.html.tostring(tree, encoding='utf-8', method='html')
+
+        # post processing to fix errors
+        content = content[5:-6]
+
+        # FIXME this is for block quotes, doesn't work for double ">"
+        content = re.sub(r"(\n&gt;)", "\n>", content)
+        content = re.sub(r"(^&gt;)", ">", content)
+
+        content = re.sub(r"```(.*?)```", unescape_repl, content, flags=re.DOTALL)
+
+        filename = self.cname_to_filename(to_canonical(name))
+        f = open(self.path + "/" + filename, 'w')
+        f.write(content)
+        f.close()
+
+        if create:
+            self.repo.add(filename)
+
+        if not message:
+            message = "Updated %s" % name
+
+        if not username:
+            username = self.default_committer_name
+
+        if not email:
+            email = "%s@realms.io" % username
+
+        return self.repo.commit(name=username,
+                                email=email,
+                                message=message,
+                                files=[filename])
+
+    def rename_page(self, old_name, new_name):
+        old_name, new_name = map(self.cname_to_filename, [old_name, new_name])
+        self.repo.mv([(old_name, new_name)])
+        self.repo.commit(name=self.default_committer_name,
+                         email=self.default_committer_email,
+                         message="Moving %s to %s" % (old_name, new_name),
+                         files=[old_name])
+
+    def get_page(self, name, sha='HEAD'):
+        # commit = gittle.utils.git.commit_info(self.repo[sha])
+        name = self.cname_to_filename(name)
+        try:
+            return self.repo.get_commit_files(sha, paths=[name]).get(name)
+        except KeyError:
+            # HEAD doesn't exist yet
+            return None
+
+    def compare(self, name, old_sha, new_sha):
+        old = self.get_page(name, sha=old_sha)
+        new = self.get_page(name, sha=new_sha)
+        return ghdiff.diff(old['data'], new['data'])
+
+    def get_history(self, name):
+        return self.repo.file_history(self.cname_to_filename(name))
+
+    def cname_to_filename(self, cname):
+        return cname.lower() + ".md"