Personal/realms-wiki
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

WIP

Browse source
This commit is contained in:
Matthew Scragg 2014-08-30 10:06:12 -05:00
parent b02d3db684
commit 86f0549e44
24 changed files with 710 additions and 398 deletions
Download patch file Download diff file Expand all files Collapse all files

11
realms/lib/assets.py
View file

@ -1,11 +0,0 @@
from flask.ext.assets import Bundle, Environment
# This can be done better, make it better
assets = Environment()
filters = 'uglifyjs'
output = 'assets/%(version)s.js'
def register(name, *files):
assets.register(name, Bundle(*files, filters=filters, output=output))

287
realms/lib/model.py Normal file
View file

@ -0,0 +1,287 @@
import json
from realms import db
from sqlalchemy import not_
from datetime import datetime
class Model(db.Model):
"""Base SQLAlchemy Model for automatic serialization and
deserialization of columns and nested relationships.
Source: https://gist.github.com/alanhamlett/6604662
Usage::
>>> class User(Model):
>>> id = db.Column(db.Integer(), primary_key=True)
>>> email = db.Column(db.String(), index=True)
>>> name = db.Column(db.String())
>>> password = db.Column(db.String())
>>> posts = db.relationship('Post', backref='user', lazy='dynamic')
>>> ...
>>> default_fields = ['email', 'name']
>>> hidden_fields = ['password']
>>> readonly_fields = ['email', 'password']
>>>
>>> class Post(Model):
>>> id = db.Column(db.Integer(), primary_key=True)
>>> user_id = db.Column(db.String(), db.ForeignKey('user.id'), nullable=False)
>>> title = db.Column(db.String())
>>> ...
>>> default_fields = ['title']
>>> readonly_fields = ['user_id']
>>>
>>> model = User(email='john@localhost')
>>> db.session.add(model)
>>> db.session.commit()
>>>
>>> # update name and create a new post
>>> validated_input = {'name': 'John', 'posts': [{'title':'My First Post'}]}
>>> model.set_columns(**validated_input)
>>> db.session.commit()
>>>
>>> print(model.to_dict(show=['password', 'posts']))
>>> {u'email': u'john@localhost', u'posts': [{u'id': 1, u'title': u'My First Post'}], u'name': u'John', u'id': 1}
"""
__abstract__ = True
# Stores changes made to this model's attributes. Can be retrieved
# with model.changes
_changes = {}
def __init__(self, **kwargs):
kwargs['_force'] = True
self._set_columns(**kwargs)
def _set_columns(self, **kwargs):
force = kwargs.get('_force')
readonly = []
if hasattr(self, 'readonly_fields'):
readonly = self.readonly_fields
if hasattr(self, 'hidden_fields'):
readonly += self.hidden_fields
readonly += [
'id',
'created',
'updated',
'modified',
'created_at',
'updated_at',
'modified_at',
]
changes = {}
columns = self.__table__.columns.keys()
relationships = self.__mapper__.relationships.keys()
for key in columns:
allowed = True if force or key not in readonly else False
exists = True if key in kwargs else False
if allowed and exists:
val = getattr(self, key)
if val != kwargs[key]:
changes[key] = {'old': val, 'new': kwargs[key]}
setattr(self, key, kwargs[key])
for rel in relationships:
allowed = True if force or rel not in readonly else False
exists = True if rel in kwargs else False
if allowed and exists:
is_list = self.__mapper__.relationships[rel].uselist
if is_list:
valid_ids = []
query = getattr(self, rel)
cls = self.__mapper__.relationships[rel].argument()
for item in kwargs[rel]:
if 'id' in item and query.filter_by(id=item['id']).limit(1).count() == 1:
obj = cls.query.filter_by(id=item['id']).first()
col_changes = obj.set_columns(**item)
if col_changes:
col_changes['id'] = str(item['id'])
if rel in changes:
changes[rel].append(col_changes)
else:
changes.update({rel: [col_changes]})
valid_ids.append(str(item['id']))
else:
col = cls()
col_changes = col.set_columns(**item)
query.append(col)
db.session.flush()
if col_changes:
col_changes['id'] = str(col.id)
if rel in changes:
changes[rel].append(col_changes)
else:
changes.update({rel: [col_changes]})
valid_ids.append(str(col.id))
# delete related rows that were not in kwargs[rel]
for item in query.filter(not_(cls.id.in_(valid_ids))).all():
col_changes = {
'id': str(item.id),
'deleted': True,
}
if rel in changes:
changes[rel].append(col_changes)
else:
changes.update({rel: [col_changes]})
db.session.delete(item)
else:
val = getattr(self, rel)
if self.__mapper__.relationships[rel].query_class is not None:
if val is not None:
col_changes = val.set_columns(**kwargs[rel])
if col_changes:
changes.update({rel: col_changes})
else:
if val != kwargs[rel]:
setattr(self, rel, kwargs[rel])
changes[rel] = {'old': val, 'new': kwargs[rel]}
return changes
def set_columns(self, **kwargs):
self._changes = self._set_columns(**kwargs)
if 'modified' in self.__table__.columns:
self.modified = datetime.utcnow()
if 'updated' in self.__table__.columns:
self.updated = datetime.utcnow()
if 'modified_at' in self.__table__.columns:
self.modified_at = datetime.utcnow()
if 'updated_at' in self.__table__.columns:
self.updated_at = datetime.utcnow()
return self._changes
def __repr__(self):
if 'id' in self.__table__.columns.keys():
return '%s(%s)' % (self.__class__.__name__, self.id)
data = {}
for key in self.__table__.columns.keys():
val = getattr(self, key)
if type(val) is datetime:
val = val.strftime('%Y-%m-%dT%H:%M:%SZ')
data[key] = val
return json.dumps(data, use_decimal=True)
@property
def changes(self):
return self._changes
def reset_changes(self):
self._changes = {}
def to_dict(self, show=None, hide=None, path=None, show_all=None):
""" Return a dictionary representation of this model.
"""
if not show:
show = []
if not hide:
hide = []
hidden = []
if hasattr(self, 'hidden_fields'):
hidden = self.hidden_fields
default = []
if hasattr(self, 'default_fields'):
default = self.default_fields
ret_data = {}
if not path:
path = self.__tablename__.lower()
def prepend_path(item):
item = item.lower()
if item.split('.', 1)[0] == path:
return item
if len(item) == 0:
return item
if item[0] != '.':
item = '.%s' % item
item = '%s%s' % (path, item)
return item
show[:] = [prepend_path(x) for x in show]
hide[:] = [prepend_path(x) for x in hide]
columns = self.__table__.columns.keys()
relationships = self.__mapper__.relationships.keys()
properties = dir(self)
for key in columns:
check = '%s.%s' % (path, key)
if check in hide or key in hidden:
continue
if show_all or key is 'id' or check in show or key in default:
ret_data[key] = getattr(self, key)
for key in relationships:
check = '%s.%s' % (path, key)
if check in hide or key in hidden:
continue
if show_all or check in show or key in default:
hide.append(check)
is_list = self.__mapper__.relationships[key].uselist
if is_list:
ret_data[key] = []
for item in getattr(self, key):
ret_data[key].append(item.to_dict(
show=show,
hide=hide,
path=('%s.%s' % (path, key.lower())),
show_all=show_all,
))
else:
if self.__mapper__.relationships[key].query_class is not None:
ret_data[key] = getattr(self, key).to_dict(
show=show,
hide=hide,
path=('%s.%s' % (path, key.lower())),
show_all=show_all,
)
else:
ret_data[key] = getattr(self, key)
for key in list(set(properties) - set(columns) - set(relationships)):
if key.startswith('_'):
continue
check = '%s.%s' % (path, key)
if check in hide or key in hidden:
continue
if show_all or check in show or key in default:
val = getattr(self, key)
try:
ret_data[key] = json.loads(json.dumps(val))
except:
pass
return ret_data
@classmethod
def insert_or_update(cls, cond, data):
obj = cls.query.filter_by(**cond).first()
if obj:
obj.set_columns(**data)
else:
data.update(cond)
obj = cls(**data)
db.session.add(obj)
db.session.commit()
def save(self):
if self not in db.session:
db.session.merge(self)
db.session.commit()
def delete(self):
if self not in db.session:
db.session.merge(self)
db.session.delete(self)
db.session.commit()
@classmethod
def get_by_id(cls, id_):
return cls.query.filter_by(id=id_).first()

46
realms/lib/ratelimit.py
View file

@ -1,46 +0,0 @@
import time
from functools import update_wrapper
from flask import request, g
from services import db
class RateLimit(object):
expiration_window = 10
def __init__(self, key_prefix, limit, per, send_x_headers):
self.reset = (int(time.time()) // per) * per + per
self.key = key_prefix + str(self.reset)
self.limit = limit
self.per = per
self.send_x_headers = send_x_headers
p = db.pipeline()
p.incr(self.key)
p.expireat(self.key, self.reset + self.expiration_window)
self.current = min(p.execute()[0], limit)
remaining = property(lambda x: x.limit - x.current)
over_limit = property(lambda x: x.current >= x.limit)
def get_view_rate_limit():
return getattr(g, '_view_rate_limit', None)
def on_over_limit(limit):
return 'Slow it down', 400
def ratelimiter(limit, per=300, send_x_headers=True,
over_limit=on_over_limit,
scope_func=lambda: request.remote_addr,
key_func=lambda: request.endpoint):
def decorator(f):
def rate_limited(*args, **kwargs):
key = 'rate-limit/%s/%s/' % (key_func(), scope_func())
rlimit = RateLimit(key, limit, per, send_x_headers)
g._view_rate_limit = rlimit
if over_limit is not None and rlimit.over_limit:
return over_limit(rlimit)
return f(*args, **kwargs)
return update_wrapper(rate_limited, f)
return decorator

4
realms/lib/services.py
View file

@ -1,4 +0,0 @@
import redis
from realms import config
db = redis.StrictRedis(host=config.REDIS_HOST, port=config.REDIS_PORT, db=config.REDIS_DB)

64
realms/lib/session.py
View file

@ -1,64 +0,0 @@
import pickle
from datetime import timedelta
from uuid import uuid4
from redis import Redis
from werkzeug.datastructures import CallbackDict
from flask.sessions import SessionInterface, SessionMixin
class RedisSession(CallbackDict, SessionMixin):
def __init__(self, initial=None, sid=None, new=False):
def on_update(self):
self.modified = True
CallbackDict.__init__(self, initial, on_update)
self.sid = sid
self.new = new
self.modified = False
class RedisSessionInterface(SessionInterface):
serializer = pickle
session_class = RedisSession
def __init__(self, redis=None, prefix='session:'):
if redis is None:
redis = Redis()
self.redis = redis
self.prefix = prefix
def generate_sid(self):
return str(uuid4())
def get_redis_expiration_time(self, app, session):
if session.permanent:
return app.permanent_session_lifetime
return timedelta(days=1)
def open_session(self, app, request):
sid = request.cookies.get(app.session_cookie_name)
if not sid:
sid = self.generate_sid()
return self.session_class(sid=sid)
val = self.redis.get(self.prefix + sid)
if val is not None:
data = self.serializer.loads(val)
return self.session_class(data, sid=sid)
return self.session_class(sid=sid, new=True)
def save_session(self, app, session, response):
domain = self.get_cookie_domain(app)
if not session:
self.redis.delete(self.prefix + session.sid)
if session.modified:
response.delete_cookie(app.session_cookie_name,
domain=domain)
return
redis_exp = self.get_redis_expiration_time(app, session)
cookie_exp = self.get_expiration_time(app, session)
val = self.serializer.dumps(dict(session))
self.redis.setex(self.prefix + session.sid, val,
int(redis_exp.total_seconds()))
response.set_cookie(app.session_cookie_name, session.sid,
expires=cookie_exp, httponly=True,
domain=domain)

32
realms/lib/util.py
View file

@ -3,8 +3,6 @@ import os
import hashlib
import json
from realms.lib.services import db
class AttrDict(dict):
def __init__(self, *args, **kwargs):
@ -34,36 +32,6 @@ def to_dict(data):
return row2dict(data)
def cache_it(fn):
def wrap(*args, **kw):
key = "%s:%s" % (args[0].table, args[1])
data = db.get(key)
# Assume strings are JSON encoded
try:
data = json.loads(data)
except TypeError:
pass
except ValueError:
pass
if data is not None:
return data
else:
data = fn(*args)
print data
ret = data
if data is None:
data = ''
if not isinstance(data, basestring):
try:
data = json.dumps(data, separators=(',', ':'))
except TypeError:
pass
db.set(key, data)
return ret
return wrap
def mkdir_safe(path):
if path and not(os.path.exists(path)):
os.makedirs(path)

155
realms/lib/wiki.py
View file

@ -1,155 +0,0 @@
import os
import re
import lxml.html
from lxml.html.clean import Cleaner
import ghdiff
import gittle.utils
from gittle import Gittle
from dulwich.repo import NotGitRepository
from werkzeug.utils import escape, unescape
from util import to_canonical
class MyGittle(Gittle):
def file_history(self, path):
"""Returns all commits where given file was modified
"""
versions = []
commits_info = self.commit_info()
seen_shas = set()
for commit in commits_info:
try:
files = self.get_commit_files(commit['sha'], paths=[path])
file_path, file_data = files.items()[0]
except IndexError:
continue
file_sha = file_data['sha']
if file_sha in seen_shas:
continue
else:
seen_shas.add(file_sha)
versions.append(dict(author=commit['author']['name'],
time=commit['time'],
file_sha=file_sha,
sha=commit['sha'],
message=commit['message']))
return versions
def mv_fs(self, file_pair):
old_name, new_name = file_pair
os.rename(self.path + "/" + old_name, self.path + "/" + new_name)
class Wiki():
path = None
base_path = '/'
default_ref = 'master'
default_committer_name = 'Anon'
default_committer_email = 'anon@anon.anon'
index_page = 'home'
repo = None
def __init__(self, path):
try:
self.repo = MyGittle(path)
except NotGitRepository:
self.repo = MyGittle.init(path)
self.path = path
def revert_page(self, name, commit_sha, message, username):
page = self.get_page(name, commit_sha)
if not page:
# Page not found
return None
commit_info = gittle.utils.git.commit_info(self.repo[commit_sha.encode('latin-1')])
message = commit_info['message']
return self.write_page(name, page['data'], message=message, username=username)
def write_page(self, name, content, message=None, create=False, username=None, email=None):
def escape_repl(m):
if m.group(1):
return "```" + escape(m.group(1)) + "```"
def unescape_repl(m):
if m.group(1):
return "```" + unescape(m.group(1)) + "```"
# prevents p tag from being added, we remove this later
content = '<div>' + content + '</div>'
content = re.sub(r"```(.*?)```", escape_repl, content, flags=re.DOTALL)
tree = lxml.html.fromstring(content)
cleaner = Cleaner(remove_unknown_tags=False,
kill_tags=set(['style']),
safe_attrs_only=False)
tree = cleaner.clean_html(tree)
content = lxml.html.tostring(tree, encoding='utf-8', method='html')
# post processing to fix errors
content = content[5:-6]
# FIXME this is for block quotes, doesn't work for double ">"
content = re.sub(r"(\n&gt;)", "\n>", content)
content = re.sub(r"(^&gt;)", ">", content)
content = re.sub(r"```(.*?)```", unescape_repl, content, flags=re.DOTALL)
filename = self.cname_to_filename(to_canonical(name))
with open(self.path + "/" + filename, 'w') as f:
f.write(content)
if create:
self.repo.add(filename)
if not message:
message = "Updated %s" % name
if not username:
username = self.default_committer_name
if not email:
email = self.default_committer_email
return self.repo.commit(name=username,
email=email,
message=message,
files=[filename])
def rename_page(self, old_name, new_name):
old_name, new_name = map(self.cname_to_filename, [old_name, new_name])
self.repo.mv([(old_name, new_name)])
self.repo.commit(name=self.default_committer_name,
email=self.default_committer_email,
message="Moving %s to %s" % (old_name, new_name),
files=[old_name])
def get_page(self, name, sha='HEAD'):
# commit = gittle.utils.git.commit_info(self.repo[sha])
name = self.cname_to_filename(name).encode('latin-1')
sha = sha.encode('latin-1')
try:
return self.repo.get_commit_files(sha, paths=[name]).get(name)
except KeyError:
# HEAD doesn't exist yet
return None
def compare(self, name, old_sha, new_sha):
old = self.get_page(name, sha=old_sha)
new = self.get_page(name, sha=new_sha)
return ghdiff.diff(old['data'], new['data'])
def get_history(self, name):
return self.repo.file_history(self.cname_to_filename(name))
def cname_to_filename(self, cname):
return cname.lower() + ".md"