diff --git a/docker/Dockerfile b/docker/Dockerfile
index f805e2d..3a5ef95 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -28,7 +28,6 @@ ENV GEVENT_RESOLVER=ares
ENV REALMS_ENV=docker
ENV REALMS_WIKI_PATH=/home/wiki/data/repo
ENV REALMS_DB_URI='sqlite:////home/wiki/data/wiki.db'
-ENV REALMS_SQLALCHEMY_DATABASE_URI=${REALMS_DB_URI}
RUN mkdir /home/wiki/data && touch /home/wiki/data/.a
VOLUME /home/wiki/data
diff --git a/realms/__init__.py b/realms/__init__.py
index df06208..8c4c571 100644
--- a/realms/__init__.py
+++ b/realms/__init__.py
@@ -1,3 +1,4 @@
+import functools
import sys
# Set default encoding to UTF-8
@@ -75,7 +76,9 @@ class Application(Flask):
# Click
if hasattr(sources, 'commands'):
- cli.add_command(sources.commands.cli, name=module_name)
+ if sources.commands.cli.name == 'cli':
+ sources.commands.cli.name = module_name
+ cli.add_command(sources.commands.cli)
# Hooks
if hasattr(sources, 'hooks'):
@@ -287,9 +290,8 @@ class AppGroup(click.Group):
kwargs.setdefault('cls', AppGroup)
return click.Group.group(self, *args, **kwargs)
-flask_cli = AppGroup()
+cli = AppGroup()
+# Decorator to be used in modules instead of click.group
+cli_group = functools.partial(click.group, cls=AppGroup)
-@flask_cli.group()
-def cli():
- pass
diff --git a/realms/commands.py b/realms/commands.py
index acb50fb..ac228c9 100644
--- a/realms/commands.py
+++ b/realms/commands.py
@@ -1,4 +1,4 @@
-from realms import config, create_app, db, __version__, flask_cli as cli, cache
+from realms import config, create_app, db, __version__, cli, cache
from realms.lib.util import random_string, in_virtualenv, green, yellow, red
from subprocess import call, Popen
from multiprocessing import cpu_count
diff --git a/realms/config/__init__.py b/realms/config/__init__.py
index 8a3f0b0..4150a3d 100644
--- a/realms/config/__init__.py
+++ b/realms/config/__init__.py
@@ -23,7 +23,7 @@ class Config(object):
BASE_URL = 'http://localhost'
SITE_TITLE = "Realms"
- # https://pythonhosted.org/Flask-SQLAlchemy/config.html#connection-uri-format
+ # http://flask-sqlalchemy.pocoo.org/config/#connection-uri-format
DB_URI = 'sqlite:////tmp/wiki.db'
# DB_URI = 'mysql://scott:tiger@localhost/mydatabase'
# DB_URI = 'postgresql://scott:tiger@localhost/mydatabase'
diff --git a/realms/lib/hook.py b/realms/lib/hook.py
index 61d2885..0327263 100644
--- a/realms/lib/hook.py
+++ b/realms/lib/hook.py
@@ -25,9 +25,15 @@ class HookMixinMeta(type):
def __new__(cls, name, bases, attrs):
super_new = super(HookMixinMeta, cls).__new__
+ hookable = []
for key, value in attrs.items():
+ # Disallow hooking methods which start with an underscore (allow __init__ etc. still)
+ if key.startswith('_') and not key.startswith('__'):
+ continue
if callable(value):
attrs[key] = hook_func(key, value)
+ hookable.append(key)
+ attrs['_hookable'] = hookable
return super_new(cls, name, bases, attrs)
@@ -37,9 +43,12 @@ class HookMixin(object):
_pre_hooks = {}
_post_hooks = {}
+ _hookable = []
@classmethod
def after(cls, method_name):
+ assert method_name in cls._hookable, "'%s' not a hookable method of '%s'" % (method_name, cls.__name__)
+
def outer(f, *args, **kwargs):
cls._post_hooks.setdefault(method_name, []).append((f, args, kwargs))
return f
@@ -47,6 +56,8 @@ class HookMixin(object):
@classmethod
def before(cls, method_name):
+ assert method_name in cls._hookable, "'%s' not a hookable method of '%s'" % (method_name, cls.__name__)
+
def outer(f, *args, **kwargs):
cls._pre_hooks.setdefault(method_name, []).append((f, args, kwargs))
return f
diff --git a/realms/modules/auth/local/commands.py b/realms/modules/auth/local/commands.py
index 20183be..ae4a7e9 100644
--- a/realms/modules/auth/local/commands.py
+++ b/realms/modules/auth/local/commands.py
@@ -2,10 +2,10 @@ import click
from realms.lib.util import random_string
from realms.modules.auth.local.models import User
from realms.lib.util import green, red, yellow
-from realms import flask_cli
+from realms import cli_group
-@flask_cli.group(short_help="Auth Module")
+@cli_group(short_help="Auth Module")
def cli():
pass
diff --git a/realms/modules/auth/oauth/models.py b/realms/modules/auth/oauth/models.py
index 72f0470..d7f7680 100644
--- a/realms/modules/auth/oauth/models.py
+++ b/realms/modules/auth/oauth/models.py
@@ -40,7 +40,7 @@ providers = {
'field_map': {
'id': 'id',
'username': 'login',
- 'email': 'email'
+ 'email': lambda(data): data.get('email') or data['login'] + '@users.noreply.github.com'
},
'token_name': 'access_token'
},
@@ -118,6 +118,8 @@ class User(BaseUser):
def get_value(d, key):
if isinstance(key, basestring):
return d.get(key)
+ elif callable(key):
+ return key(d)
# key should be list here
val = d.get(key.pop(0))
if len(key) == 0:
diff --git a/realms/modules/search/commands.py b/realms/modules/search/commands.py
index 43507df..eab1556 100644
--- a/realms/modules/search/commands.py
+++ b/realms/modules/search/commands.py
@@ -1,10 +1,10 @@
import click
-from realms import create_app, search, flask_cli
+from flask import current_app
+from realms import search, cli_group
from realms.modules.wiki.models import Wiki
-from realms.lib.util import filename_to_cname
-@flask_cli.group(short_help="Search Module")
+@cli_group(short_help="Search Module")
def cli():
pass
@@ -13,27 +13,25 @@ def cli():
def rebuild_index():
""" Rebuild search index
"""
- app = create_app()
-
- if app.config.get('SEARCH_TYPE') == 'simple':
+ if current_app.config.get('SEARCH_TYPE') == 'simple':
click.echo("Search type is simple, try using elasticsearch.")
return
- with app.app_context():
- # Wiki
- search.delete_index('wiki')
- wiki = Wiki(app.config['WIKI_PATH'])
- for entry in wiki.get_index():
- page = wiki.get_page(entry['name'])
- if not page:
- # Some non-markdown files may have issues
- continue
- name = filename_to_cname(page['path'])
- # TODO add email?
- body = dict(name=name,
- content=page.data,
- message=page.info['message'],
- username=page.info['author'],
- updated_on=entry['mtime'],
- created_on=entry['ctime'])
- search.index_wiki(name, body)
+ # Wiki
+ search.delete_index('wiki')
+ wiki = Wiki(current_app.config['WIKI_PATH'])
+ for entry in wiki.get_index():
+ page = wiki.get_page(entry['name'])
+ if not page:
+ # Some non-markdown files may have issues
+ continue
+ # TODO add email?
+ # TODO I have concens about indexing the commit info from latest revision, see #148
+ info = next(page.history)
+ body = dict(name=page.name,
+ content=page.data,
+ message=info['message'],
+ username=info['author'],
+ updated_on=entry['mtime'],
+ created_on=entry['ctime'])
+ search.index_wiki(page.name, body)
diff --git a/realms/modules/wiki/models.py b/realms/modules/wiki/models.py
index e23632a..b32f076 100644
--- a/realms/modules/wiki/models.py
+++ b/realms/modules/wiki/models.py
@@ -104,47 +104,87 @@ class WikiPage(HookMixin):
return data
@property
- def info(self):
- cache_key = self._cache_key('info')
- cached = cache.get(cache_key)
- if cached:
- return cached
-
- info = self.get_history(limit=1)[0]
- cache.set(cache_key, info)
- return info
-
- def get_history(self, limit=100):
+ def history(self):
"""Get page history.
- :param limit: Limit history size.
- :return: list -- List of dicts
+ History can take a long time to generate for repositories with many commits.
+ This returns an iterator to avoid having to load them all at once, and caches
+ as it goes.
+
+ :return: iter -- Iterator over dicts
"""
- versions = []
-
- try:
- walker = self.wiki.repo.get_walker(paths=[self.filename], max_entries=limit, follow=True)
- except KeyError:
- # We don't have a head, no commits
- return []
+ cache_head = []
+ cache_tail = cache.get(self._cache_key('history')) or [{'_cache_missing': True}]
+ while True:
+ if not cache_tail:
+ return
+ for index, cached_rev in enumerate(cache_tail):
+ if cached_rev.get("_cache_missing"):
+ break
+ else:
+ yield cached_rev
+ cache_head.extend(cache_tail[:index])
+ cache_tail = cache_tail[index+1:]
+ start_sha = cached_rev.get('sha')
+ end_sha = cache_tail[0].get('sha') if cache_tail else None
+ for rev in self._iter_revs(start_sha=start_sha, end_sha=end_sha, filename=cached_rev.get('filename')):
+ cache_head.append(rev)
+ placeholder = {
+ '_cache_missing': True,
+ 'sha': rev['sha'],
+ 'filename': rev['new_filename']
+ }
+ cache.set(self._cache_key('history'), cache_head + [placeholder] + cache_tail)
+ yield rev
+ cache.set(self._cache_key('history'), cache_head + cache_tail)
+ def _iter_revs(self, start_sha=None, end_sha=None, filename=None):
+ if end_sha:
+ end_sha = [end_sha]
+ if not len(self.wiki.repo.open_index()):
+ # Index is empty, no commits
+ return
+ filename = filename or self.filename
+ walker = iter(self.wiki.repo.get_walker(paths=[filename],
+ include=start_sha,
+ exclude=end_sha,
+ follow=True))
+ if start_sha:
+ # If we are not starting from HEAD, we already have the start commit
+ next(walker)
+ filename = self.filename
for entry in walker:
change_type = None
for change in entry.changes():
- # Changes should already be filtered to only the one affecting our file
- change_type = change.type
- break
- author_name, author_email = entry.commit.author.rstrip('>').split('<')
- versions.append(dict(
- author=author_name.strip(),
- author_email=author_email,
- time=entry.commit.author_time,
- message=entry.commit.message,
- sha=entry.commit.id,
- type=change_type))
+ if change.new.path == filename:
+ filename = change.old.path
+ change_type = change.type
+ break
- return versions
+ author_name, author_email = entry.commit.author.rstrip('>').split('<')
+ r = dict(author=author_name.strip(),
+ author_email=author_email,
+ time=entry.commit.author_time,
+ message=entry.commit.message,
+ sha=entry.commit.id,
+ type=change_type,
+ new_filename=change.new.path,
+ old_filename=change.old.path)
+ yield r
+
+ @property
+ def history_cache(self):
+ """Get info about the history cache.
+
+ :return: tuple -- (cached items, cache complete?)
+ """
+ cached_revs = cache.get(self._cache_key('history'))
+ if not cached_revs:
+ return 0, False
+ elif any(rev.get('_cache_missing') for rev in cached_revs):
+ return len(cached_revs) - 1, False
+ return len(cached_revs), True
@property
def partials(self):
@@ -191,8 +231,14 @@ class WikiPage(HookMixin):
return username, email
- def _clear_cache(self):
- cache.delete_many(*(self._cache_key(p) for p in ['data', 'info']))
+ def _invalidate_cache(self, save_history=None):
+ cache.delete(self._cache_key('data'))
+ if save_history:
+ if not save_history[0].get('_cache_missing'):
+ save_history = [{'_cache_missing': True}] + save_history
+ cache.set(self._cache_key('history'), save_history)
+ else:
+ cache.delete(self._cache_key('history'))
def delete(self, username=None, email=None, message=None):
"""Delete page.
@@ -211,7 +257,7 @@ class WikiPage(HookMixin):
email=email,
message=message,
files=[self.filename])
- self._clear_cache()
+ self._invalidate_cache()
return commit
def rename(self, new_name, username=None, email=None, message=None):
@@ -245,11 +291,12 @@ class WikiPage(HookMixin):
message=message,
files=[old_filename, new_filename])
- self._clear_cache()
+ old_history = cache.get(self._cache_key('history'))
+ self._invalidate_cache()
self.name = new_name
self.filename = new_filename
# We need to clear the cache for the new name as well as the old
- self._clear_cache()
+ self._invalidate_cache(save_history=old_history)
return commit
@@ -281,7 +328,8 @@ class WikiPage(HookMixin):
message=message,
files=[self.filename])
- self._clear_cache()
+ old_history = cache.get(self._cache_key('history'))
+ self._invalidate_cache(save_history=old_history)
return ret
def revert(self, commit_sha, message, username, email):
diff --git a/realms/modules/wiki/tests.py b/realms/modules/wiki/tests.py
index 4e9c8eb..565f4f1 100644
--- a/realms/modules/wiki/tests.py
+++ b/realms/modules/wiki/tests.py
@@ -41,7 +41,7 @@ class WikiTest(WikiBaseTest):
self.assert_200(rv)
self.assert_context('name', 'test')
- eq_(self.get_context_variable('page').info['message'], 'test message')
+ eq_(next(self.get_context_variable('page').history)['message'], 'test message')
eq_(self.get_context_variable('page').data, 'testing')
def test_history(self):
diff --git a/realms/modules/wiki/views.py b/realms/modules/wiki/views.py
index d9e7fab..c851741 100644
--- a/realms/modules/wiki/views.py
+++ b/realms/modules/wiki/views.py
@@ -1,5 +1,6 @@
import itertools
import sys
+from datetime import datetime
from flask import abort, g, render_template, request, redirect, Blueprint, flash, url_for, current_app
from flask.ext.login import login_required, current_user
from realms.lib.util import to_canonical, remove_ext, gravatar_url
@@ -64,11 +65,37 @@ def revert():
def history(name):
if current_app.config.get('PRIVATE_WIKI') and current_user.is_anonymous():
return current_app.login_manager.unauthorized()
+ return render_template('wiki/history.html', name=name)
- hist = g.current_wiki.get_page(name).get_history()
- for item in hist:
+
+@blueprint.route("/_history_data/Page Not Found
+ {% if error is defined %}
+ {{ error.description }}
+ {% endif %}
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/realms/templates/wiki/history.html b/realms/templates/wiki/history.html
index bdc0cc7..43db7c4 100644
--- a/realms/templates/wiki/history.html
+++ b/realms/templates/wiki/history.html
@@ -6,27 +6,17 @@
Compare Revisions
Name | Revision Message | Date | |
---|---|---|---|
- {% if h.type != 'delete' %} - - {% endif %} - | -{{ h.author }} | -View {{ h.message }} | -{{ h.time|datetime }} | -
Loading file history... |
Compare Revisions @@ -34,25 +24,95 @@ {% endblock %} +{% block css %} + +{% endblock %} + {% block js %}