diff --git a/docker/Dockerfile b/docker/Dockerfile index f805e2d..3a5ef95 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -28,7 +28,6 @@ ENV GEVENT_RESOLVER=ares ENV REALMS_ENV=docker ENV REALMS_WIKI_PATH=/home/wiki/data/repo ENV REALMS_DB_URI='sqlite:////home/wiki/data/wiki.db' -ENV REALMS_SQLALCHEMY_DATABASE_URI=${REALMS_DB_URI} RUN mkdir /home/wiki/data && touch /home/wiki/data/.a VOLUME /home/wiki/data diff --git a/realms/__init__.py b/realms/__init__.py index df06208..8c4c571 100644 --- a/realms/__init__.py +++ b/realms/__init__.py @@ -1,3 +1,4 @@ +import functools import sys # Set default encoding to UTF-8 @@ -75,7 +76,9 @@ class Application(Flask): # Click if hasattr(sources, 'commands'): - cli.add_command(sources.commands.cli, name=module_name) + if sources.commands.cli.name == 'cli': + sources.commands.cli.name = module_name + cli.add_command(sources.commands.cli) # Hooks if hasattr(sources, 'hooks'): @@ -287,9 +290,8 @@ class AppGroup(click.Group): kwargs.setdefault('cls', AppGroup) return click.Group.group(self, *args, **kwargs) -flask_cli = AppGroup() +cli = AppGroup() +# Decorator to be used in modules instead of click.group +cli_group = functools.partial(click.group, cls=AppGroup) -@flask_cli.group() -def cli(): - pass diff --git a/realms/commands.py b/realms/commands.py index acb50fb..ac228c9 100644 --- a/realms/commands.py +++ b/realms/commands.py @@ -1,4 +1,4 @@ -from realms import config, create_app, db, __version__, flask_cli as cli, cache +from realms import config, create_app, db, __version__, cli, cache from realms.lib.util import random_string, in_virtualenv, green, yellow, red from subprocess import call, Popen from multiprocessing import cpu_count diff --git a/realms/config/__init__.py b/realms/config/__init__.py index 8a3f0b0..4150a3d 100644 --- a/realms/config/__init__.py +++ b/realms/config/__init__.py @@ -23,7 +23,7 @@ class Config(object): BASE_URL = 'http://localhost' SITE_TITLE = "Realms" - # https://pythonhosted.org/Flask-SQLAlchemy/config.html#connection-uri-format + # http://flask-sqlalchemy.pocoo.org/config/#connection-uri-format DB_URI = 'sqlite:////tmp/wiki.db' # DB_URI = 'mysql://scott:tiger@localhost/mydatabase' # DB_URI = 'postgresql://scott:tiger@localhost/mydatabase' diff --git a/realms/lib/hook.py b/realms/lib/hook.py index 61d2885..0327263 100644 --- a/realms/lib/hook.py +++ b/realms/lib/hook.py @@ -25,9 +25,15 @@ class HookMixinMeta(type): def __new__(cls, name, bases, attrs): super_new = super(HookMixinMeta, cls).__new__ + hookable = [] for key, value in attrs.items(): + # Disallow hooking methods which start with an underscore (allow __init__ etc. still) + if key.startswith('_') and not key.startswith('__'): + continue if callable(value): attrs[key] = hook_func(key, value) + hookable.append(key) + attrs['_hookable'] = hookable return super_new(cls, name, bases, attrs) @@ -37,9 +43,12 @@ class HookMixin(object): _pre_hooks = {} _post_hooks = {} + _hookable = [] @classmethod def after(cls, method_name): + assert method_name in cls._hookable, "'%s' not a hookable method of '%s'" % (method_name, cls.__name__) + def outer(f, *args, **kwargs): cls._post_hooks.setdefault(method_name, []).append((f, args, kwargs)) return f @@ -47,6 +56,8 @@ class HookMixin(object): @classmethod def before(cls, method_name): + assert method_name in cls._hookable, "'%s' not a hookable method of '%s'" % (method_name, cls.__name__) + def outer(f, *args, **kwargs): cls._pre_hooks.setdefault(method_name, []).append((f, args, kwargs)) return f diff --git a/realms/modules/auth/local/commands.py b/realms/modules/auth/local/commands.py index 20183be..ae4a7e9 100644 --- a/realms/modules/auth/local/commands.py +++ b/realms/modules/auth/local/commands.py @@ -2,10 +2,10 @@ import click from realms.lib.util import random_string from realms.modules.auth.local.models import User from realms.lib.util import green, red, yellow -from realms import flask_cli +from realms import cli_group -@flask_cli.group(short_help="Auth Module") +@cli_group(short_help="Auth Module") def cli(): pass diff --git a/realms/modules/auth/oauth/models.py b/realms/modules/auth/oauth/models.py index 72f0470..d7f7680 100644 --- a/realms/modules/auth/oauth/models.py +++ b/realms/modules/auth/oauth/models.py @@ -40,7 +40,7 @@ providers = { 'field_map': { 'id': 'id', 'username': 'login', - 'email': 'email' + 'email': lambda(data): data.get('email') or data['login'] + '@users.noreply.github.com' }, 'token_name': 'access_token' }, @@ -118,6 +118,8 @@ class User(BaseUser): def get_value(d, key): if isinstance(key, basestring): return d.get(key) + elif callable(key): + return key(d) # key should be list here val = d.get(key.pop(0)) if len(key) == 0: diff --git a/realms/modules/search/commands.py b/realms/modules/search/commands.py index 43507df..eab1556 100644 --- a/realms/modules/search/commands.py +++ b/realms/modules/search/commands.py @@ -1,10 +1,10 @@ import click -from realms import create_app, search, flask_cli +from flask import current_app +from realms import search, cli_group from realms.modules.wiki.models import Wiki -from realms.lib.util import filename_to_cname -@flask_cli.group(short_help="Search Module") +@cli_group(short_help="Search Module") def cli(): pass @@ -13,27 +13,25 @@ def cli(): def rebuild_index(): """ Rebuild search index """ - app = create_app() - - if app.config.get('SEARCH_TYPE') == 'simple': + if current_app.config.get('SEARCH_TYPE') == 'simple': click.echo("Search type is simple, try using elasticsearch.") return - with app.app_context(): - # Wiki - search.delete_index('wiki') - wiki = Wiki(app.config['WIKI_PATH']) - for entry in wiki.get_index(): - page = wiki.get_page(entry['name']) - if not page: - # Some non-markdown files may have issues - continue - name = filename_to_cname(page['path']) - # TODO add email? - body = dict(name=name, - content=page.data, - message=page.info['message'], - username=page.info['author'], - updated_on=entry['mtime'], - created_on=entry['ctime']) - search.index_wiki(name, body) + # Wiki + search.delete_index('wiki') + wiki = Wiki(current_app.config['WIKI_PATH']) + for entry in wiki.get_index(): + page = wiki.get_page(entry['name']) + if not page: + # Some non-markdown files may have issues + continue + # TODO add email? + # TODO I have concens about indexing the commit info from latest revision, see #148 + info = next(page.history) + body = dict(name=page.name, + content=page.data, + message=info['message'], + username=info['author'], + updated_on=entry['mtime'], + created_on=entry['ctime']) + search.index_wiki(page.name, body) diff --git a/realms/modules/wiki/models.py b/realms/modules/wiki/models.py index e23632a..b32f076 100644 --- a/realms/modules/wiki/models.py +++ b/realms/modules/wiki/models.py @@ -104,47 +104,87 @@ class WikiPage(HookMixin): return data @property - def info(self): - cache_key = self._cache_key('info') - cached = cache.get(cache_key) - if cached: - return cached - - info = self.get_history(limit=1)[0] - cache.set(cache_key, info) - return info - - def get_history(self, limit=100): + def history(self): """Get page history. - :param limit: Limit history size. - :return: list -- List of dicts + History can take a long time to generate for repositories with many commits. + This returns an iterator to avoid having to load them all at once, and caches + as it goes. + + :return: iter -- Iterator over dicts """ - versions = [] - - try: - walker = self.wiki.repo.get_walker(paths=[self.filename], max_entries=limit, follow=True) - except KeyError: - # We don't have a head, no commits - return [] + cache_head = [] + cache_tail = cache.get(self._cache_key('history')) or [{'_cache_missing': True}] + while True: + if not cache_tail: + return + for index, cached_rev in enumerate(cache_tail): + if cached_rev.get("_cache_missing"): + break + else: + yield cached_rev + cache_head.extend(cache_tail[:index]) + cache_tail = cache_tail[index+1:] + start_sha = cached_rev.get('sha') + end_sha = cache_tail[0].get('sha') if cache_tail else None + for rev in self._iter_revs(start_sha=start_sha, end_sha=end_sha, filename=cached_rev.get('filename')): + cache_head.append(rev) + placeholder = { + '_cache_missing': True, + 'sha': rev['sha'], + 'filename': rev['new_filename'] + } + cache.set(self._cache_key('history'), cache_head + [placeholder] + cache_tail) + yield rev + cache.set(self._cache_key('history'), cache_head + cache_tail) + def _iter_revs(self, start_sha=None, end_sha=None, filename=None): + if end_sha: + end_sha = [end_sha] + if not len(self.wiki.repo.open_index()): + # Index is empty, no commits + return + filename = filename or self.filename + walker = iter(self.wiki.repo.get_walker(paths=[filename], + include=start_sha, + exclude=end_sha, + follow=True)) + if start_sha: + # If we are not starting from HEAD, we already have the start commit + next(walker) + filename = self.filename for entry in walker: change_type = None for change in entry.changes(): - # Changes should already be filtered to only the one affecting our file - change_type = change.type - break - author_name, author_email = entry.commit.author.rstrip('>').split('<') - versions.append(dict( - author=author_name.strip(), - author_email=author_email, - time=entry.commit.author_time, - message=entry.commit.message, - sha=entry.commit.id, - type=change_type)) + if change.new.path == filename: + filename = change.old.path + change_type = change.type + break - return versions + author_name, author_email = entry.commit.author.rstrip('>').split('<') + r = dict(author=author_name.strip(), + author_email=author_email, + time=entry.commit.author_time, + message=entry.commit.message, + sha=entry.commit.id, + type=change_type, + new_filename=change.new.path, + old_filename=change.old.path) + yield r + + @property + def history_cache(self): + """Get info about the history cache. + + :return: tuple -- (cached items, cache complete?) + """ + cached_revs = cache.get(self._cache_key('history')) + if not cached_revs: + return 0, False + elif any(rev.get('_cache_missing') for rev in cached_revs): + return len(cached_revs) - 1, False + return len(cached_revs), True @property def partials(self): @@ -191,8 +231,14 @@ class WikiPage(HookMixin): return username, email - def _clear_cache(self): - cache.delete_many(*(self._cache_key(p) for p in ['data', 'info'])) + def _invalidate_cache(self, save_history=None): + cache.delete(self._cache_key('data')) + if save_history: + if not save_history[0].get('_cache_missing'): + save_history = [{'_cache_missing': True}] + save_history + cache.set(self._cache_key('history'), save_history) + else: + cache.delete(self._cache_key('history')) def delete(self, username=None, email=None, message=None): """Delete page. @@ -211,7 +257,7 @@ class WikiPage(HookMixin): email=email, message=message, files=[self.filename]) - self._clear_cache() + self._invalidate_cache() return commit def rename(self, new_name, username=None, email=None, message=None): @@ -245,11 +291,12 @@ class WikiPage(HookMixin): message=message, files=[old_filename, new_filename]) - self._clear_cache() + old_history = cache.get(self._cache_key('history')) + self._invalidate_cache() self.name = new_name self.filename = new_filename # We need to clear the cache for the new name as well as the old - self._clear_cache() + self._invalidate_cache(save_history=old_history) return commit @@ -281,7 +328,8 @@ class WikiPage(HookMixin): message=message, files=[self.filename]) - self._clear_cache() + old_history = cache.get(self._cache_key('history')) + self._invalidate_cache(save_history=old_history) return ret def revert(self, commit_sha, message, username, email): diff --git a/realms/modules/wiki/tests.py b/realms/modules/wiki/tests.py index 4e9c8eb..565f4f1 100644 --- a/realms/modules/wiki/tests.py +++ b/realms/modules/wiki/tests.py @@ -41,7 +41,7 @@ class WikiTest(WikiBaseTest): self.assert_200(rv) self.assert_context('name', 'test') - eq_(self.get_context_variable('page').info['message'], 'test message') + eq_(next(self.get_context_variable('page').history)['message'], 'test message') eq_(self.get_context_variable('page').data, 'testing') def test_history(self): diff --git a/realms/modules/wiki/views.py b/realms/modules/wiki/views.py index d9e7fab..c851741 100644 --- a/realms/modules/wiki/views.py +++ b/realms/modules/wiki/views.py @@ -1,5 +1,6 @@ import itertools import sys +from datetime import datetime from flask import abort, g, render_template, request, redirect, Blueprint, flash, url_for, current_app from flask.ext.login import login_required, current_user from realms.lib.util import to_canonical, remove_ext, gravatar_url @@ -64,11 +65,37 @@ def revert(): def history(name): if current_app.config.get('PRIVATE_WIKI') and current_user.is_anonymous(): return current_app.login_manager.unauthorized() + return render_template('wiki/history.html', name=name) - hist = g.current_wiki.get_page(name).get_history() - for item in hist: + +@blueprint.route("/_history_data/") +def history_data(name): + """Ajax provider for paginated history data.""" + if current_app.config.get('PRIVATE_WIKI') and current_user.is_anonymous(): + return current_app.login_manager.unauthorized() + draw = int(request.args.get('draw', 0)) + start = int(request.args.get('start', 0)) + length = int(request.args.get('length', 10)) + page = g.current_wiki.get_page(name) + items = list(itertools.islice(page.history, start, start + length)) + for item in items: item['gravatar'] = gravatar_url(item['author_email']) - return render_template('wiki/history.html', name=name, history=hist) + item['DT_RowId'] = item['sha'] + date = datetime.fromtimestamp(item['time']) + item['date'] = date.strftime(current_app.config.get('DATETIME_FORMAT', '%b %d, %Y %I:%M %p')) + item['link'] = url_for('.commit', name=name, sha=item['sha']) + total_records, hist_complete = page.history_cache + if not hist_complete: + # Force datatables to fetch more data when it gets to the end + total_records += 1 + return { + 'draw': draw, + 'recordsTotal': total_records, + 'recordsFiltered': total_records, + 'data': items, + 'fully_loaded': hist_complete + } + @blueprint.route("/_edit/") @@ -85,7 +112,8 @@ def edit(name): return render_template('wiki/edit.html', name=cname, content=page.data, - info=page.info, + # TODO: Remove this? See #148 + info=next(page.history), sha=page.sha, partials=page.partials) diff --git a/realms/templates/errors/404.html b/realms/templates/errors/404.html index 66bfd47..11e4044 100644 --- a/realms/templates/errors/404.html +++ b/realms/templates/errors/404.html @@ -2,5 +2,8 @@ {% block body %}

Page Not Found

+ {% if error is defined %} +

{{ error.description }}

+ {% endif %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/realms/templates/wiki/history.html b/realms/templates/wiki/history.html index bdc0cc7..43db7c4 100644 --- a/realms/templates/wiki/history.html +++ b/realms/templates/wiki/history.html @@ -6,27 +6,17 @@ Compare Revisions

- +
- - {% for h in history %} - - - - - - - {% endfor %} + + +
Name Revision Message Date
- {% if h.type != 'delete' %} - - {% endif %} - {{ h.author }}View {{ h.message }} {{ h.time|datetime }}
Loading file history...

Compare Revisions @@ -34,25 +24,95 @@ {% endblock %} +{% block css %} + +{% endblock %} + {% block js %}