diff --git a/realms/modules/search/commands.py b/realms/modules/search/commands.py index 9da87f4..eab1556 100644 --- a/realms/modules/search/commands.py +++ b/realms/modules/search/commands.py @@ -26,10 +26,12 @@ def rebuild_index(): # Some non-markdown files may have issues continue # TODO add email? + # TODO I have concens about indexing the commit info from latest revision, see #148 + info = next(page.history) body = dict(name=page.name, content=page.data, - message=page.info['message'], - username=page.info['author'], + message=info['message'], + username=info['author'], updated_on=entry['mtime'], created_on=entry['ctime']) search.index_wiki(page.name, body) diff --git a/realms/modules/wiki/models.py b/realms/modules/wiki/models.py index 318b7ae..b5273c4 100644 --- a/realms/modules/wiki/models.py +++ b/realms/modules/wiki/models.py @@ -109,47 +109,87 @@ class WikiPage(HookMixin): return data @property - def info(self): - cache_key = self._cache_key('info') - cached = cache.get(cache_key) - if cached: - return cached - - info = self.get_history(limit=1)[0] - cache.set(cache_key, info) - return info - - def get_history(self, limit=100): + def history(self): """Get page history. - :param limit: Limit history size. - :return: list -- List of dicts + History can take a long time to generate for repositories with many commits. + This returns an iterator to avoid having to load them all at once, and caches + as it goes. + + :return: iter -- Iterator over dicts """ + cache_head = [] + cache_tail = cache.get(self._cache_key('history')) or [{'_cache_missing': True}] + while True: + if not cache_tail: + return + for index, cached_rev in enumerate(cache_tail): + if cached_rev.get("_cache_missing"): + break + else: + yield cached_rev + cache_head.extend(cache_tail[:index]) + cache_tail = cache_tail[index+1:] + start_sha = cached_rev.get('sha') + end_sha = cache_tail[0].get('sha') if cache_tail else None + for rev in self._iter_revs(start_sha=start_sha, end_sha=end_sha, filename=cached_rev.get('filename')): + cache_head.append(rev) + placeholder = { + '_cache_missing': True, + 'sha': rev['sha'], + 'filename': rev['new_filename'] + } + cache.set(self._cache_key('history'), cache_head + [placeholder] + cache_tail) + yield rev + cache.set(self._cache_key('history'), cache_head + cache_tail) + + def _iter_revs(self, start_sha=None, end_sha=None, filename=None): + if end_sha: + end_sha = [end_sha] if not len(self.wiki.repo.open_index()): # Index is empty, no commits - return [] - - versions = [] - - walker = self.wiki.repo.get_walker(paths=[self.filename], max_entries=limit) + return + filename = filename or self.filename + walker = iter(self.wiki.repo.get_walker(paths=[filename], + include=start_sha, + exclude=end_sha, + follow=True)) + if start_sha: + # If we are not starting from HEAD, we already have the start commit + next(walker) + filename = self.filename for entry in walker: change_type = None for change in entry.changes(): - if change.old.path == self.filename: + if change.new.path == filename: + filename = change.old.path change_type = change.type - elif change.new.path == self.filename: - change_type = change.type - author_name, author_email = entry.commit.author.rstrip('>').split('<') - versions.append(dict( - author=author_name.strip(), - author_email=author_email, - time=entry.commit.author_time, - message=entry.commit.message, - sha=entry.commit.id, - type=change_type)) + break - return versions + author_name, author_email = entry.commit.author.rstrip('>').split('<') + r = dict(author=author_name.strip(), + author_email=author_email, + time=entry.commit.author_time, + message=entry.commit.message, + sha=entry.commit.id, + type=change_type, + new_filename=change.new.path, + old_filename=change.old.path) + yield r + + @property + def history_cache(self): + """Get info about the history cache. + + :return: tuple -- (cached items, cache complete?) + """ + cached_revs = cache.get(self._cache_key('history')) + if not cached_revs: + return 0, False + elif any(rev.get('_cache_missing') for rev in cached_revs): + return len(cached_revs) - 1, False + return len(cached_revs), True @property def partials(self): @@ -196,8 +236,14 @@ class WikiPage(HookMixin): return username, email - def _clear_cache(self): - cache.delete_many(*(self._cache_key(p) for p in ['data', 'info'])) + def _invalidate_cache(self, save_history=None): + cache.delete(self._cache_key('data')) + if save_history: + if not save_history[0].get('_cache_missing'): + save_history = [{'_cache_missing': True}] + save_history + cache.set(self._cache_key('history'), save_history) + else: + cache.delete(self._cache_key('history')) def delete(self, username=None, email=None, message=None): """Delete page. @@ -218,7 +264,7 @@ class WikiPage(HookMixin): email=email, message=message, files=[self.filename]) - self._clear_cache() + self._invalidate_cache() return commit def rename(self, new_name, username=None, email=None, message=None): @@ -256,11 +302,12 @@ class WikiPage(HookMixin): message=message, files=[old_filename, new_filename]) - self._clear_cache() + old_history = cache.get(self._cache_key('history')) + self._invalidate_cache() self.name = new_name self.filename = new_filename # We need to clear the cache for the new name as well as the old - self._clear_cache() + self._invalidate_cache(save_history=old_history) return commit @@ -296,7 +343,8 @@ class WikiPage(HookMixin): message=message, files=[self.filename]) - self._clear_cache() + old_history = cache.get(self._cache_key('history')) + self._invalidate_cache(save_history=old_history) return ret def revert(self, commit_sha, message, username, email): diff --git a/realms/modules/wiki/tests.py b/realms/modules/wiki/tests.py index 4e9c8eb..565f4f1 100644 --- a/realms/modules/wiki/tests.py +++ b/realms/modules/wiki/tests.py @@ -41,7 +41,7 @@ class WikiTest(WikiBaseTest): self.assert_200(rv) self.assert_context('name', 'test') - eq_(self.get_context_variable('page').info['message'], 'test message') + eq_(next(self.get_context_variable('page').history)['message'], 'test message') eq_(self.get_context_variable('page').data, 'testing') def test_history(self): diff --git a/realms/modules/wiki/views.py b/realms/modules/wiki/views.py index 40571b2..a5a2e40 100644 --- a/realms/modules/wiki/views.py +++ b/realms/modules/wiki/views.py @@ -1,5 +1,6 @@ import itertools import sys +from datetime import datetime from flask import abort, g, render_template, request, redirect, Blueprint, flash, url_for, current_app from flask.ext.login import login_required, current_user from realms.lib.util import to_canonical, remove_ext, gravatar_url @@ -64,11 +65,37 @@ def revert(): def history(name): if current_app.config.get('PRIVATE_WIKI') and current_user.is_anonymous(): return current_app.login_manager.unauthorized() + return render_template('wiki/history.html', name=name) - hist = g.current_wiki.get_page(name).get_history() - for item in hist: + +@blueprint.route("/_history_data/") +def history_data(name): + """Ajax provider for paginated history data.""" + if current_app.config.get('PRIVATE_WIKI') and current_user.is_anonymous(): + return current_app.login_manager.unauthorized() + draw = int(request.args.get('draw', 0)) + start = int(request.args.get('start', 0)) + length = int(request.args.get('length', 10)) + page = g.current_wiki.get_page(name) + items = list(itertools.islice(page.history, start, start + length)) + for item in items: item['gravatar'] = gravatar_url(item['author_email']) - return render_template('wiki/history.html', name=name, history=hist) + item['DT_RowId'] = item['sha'] + date = datetime.fromtimestamp(item['time']) + item['date'] = date.strftime(current_app.config.get('DATETIME_FORMAT', '%b %d, %Y %I:%M %p')) + item['link'] = url_for('.commit', name=name, sha=item['sha']) + total_records, hist_complete = page.history_cache + if not hist_complete: + # Force datatables to fetch more data when it gets to the end + total_records += 1 + return { + 'draw': draw, + 'recordsTotal': total_records, + 'recordsFiltered': total_records, + 'data': items, + 'fully_loaded': hist_complete + } + @blueprint.route("/_edit/") @@ -85,7 +112,8 @@ def edit(name): return render_template('wiki/edit.html', name=cname, content=page.data, - info=page.info, + # TODO: Remove this? See #148 + info=next(page.history), sha=page.sha, partials=page.partials) diff --git a/realms/templates/errors/404.html b/realms/templates/errors/404.html index 66bfd47..11e4044 100644 --- a/realms/templates/errors/404.html +++ b/realms/templates/errors/404.html @@ -2,5 +2,8 @@ {% block body %}

Page Not Found

+ {% if error is defined %} +

{{ error.description }}

+ {% endif %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/realms/templates/wiki/history.html b/realms/templates/wiki/history.html index bdc0cc7..43db7c4 100644 --- a/realms/templates/wiki/history.html +++ b/realms/templates/wiki/history.html @@ -6,27 +6,17 @@ Compare Revisions

- +
- - {% for h in history %} - - - - - - - {% endfor %} + + +
Name Revision Message Date
- {% if h.type != 'delete' %} - - {% endif %} - {{ h.author }}View {{ h.message }} {{ h.time|datetime }}
Loading file history...

Compare Revisions @@ -34,25 +24,95 @@ {% endblock %} +{% block css %} + +{% endblock %} + {% block js %}