Merge branch 'master' into no_gittle
# Conflicts: # realms/modules/wiki/models.py
This commit is contained in:
Chase Sterling
commit
3223e9fa65
14 changed files with 258 additions and 107 deletions
|
|
@ -2,10 +2,10 @@ import click
|
|||
from realms.lib.util import random_string
|
||||
from realms.modules.auth.local.models import User
|
||||
from realms.lib.util import green, red, yellow
|
||||
from realms import flask_cli
|
||||
from realms import cli_group
|
||||
|
||||
|
||||
@flask_cli.group(short_help="Auth Module")
|
||||
@cli_group(short_help="Auth Module")
|
||||
def cli():
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ providers = {
|
|||
'field_map': {
|
||||
'id': 'id',
|
||||
'username': 'login',
|
||||
'email': 'email'
|
||||
'email': lambda(data): data.get('email') or data['login'] + '@users.noreply.github.com'
|
||||
},
|
||||
'token_name': 'access_token'
|
||||
},
|
||||
|
|
@ -118,6 +118,8 @@ class User(BaseUser):
|
|||
def get_value(d, key):
|
||||
if isinstance(key, basestring):
|
||||
return d.get(key)
|
||||
elif callable(key):
|
||||
return key(d)
|
||||
# key should be list here
|
||||
val = d.get(key.pop(0))
|
||||
if len(key) == 0:
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import click
|
||||
from realms import create_app, search, flask_cli
|
||||
from flask import current_app
|
||||
from realms import search, cli_group
|
||||
from realms.modules.wiki.models import Wiki
|
||||
from realms.lib.util import filename_to_cname
|
||||
|
||||
|
||||
@flask_cli.group(short_help="Search Module")
|
||||
@cli_group(short_help="Search Module")
|
||||
def cli():
|
||||
pass
|
||||
|
||||
|
|
@ -13,27 +13,25 @@ def cli():
|
|||
def rebuild_index():
|
||||
""" Rebuild search index
|
||||
"""
|
||||
app = create_app()
|
||||
|
||||
if app.config.get('SEARCH_TYPE') == 'simple':
|
||||
if current_app.config.get('SEARCH_TYPE') == 'simple':
|
||||
click.echo("Search type is simple, try using elasticsearch.")
|
||||
return
|
||||
|
||||
with app.app_context():
|
||||
# Wiki
|
||||
search.delete_index('wiki')
|
||||
wiki = Wiki(app.config['WIKI_PATH'])
|
||||
for entry in wiki.get_index():
|
||||
page = wiki.get_page(entry['name'])
|
||||
if not page:
|
||||
# Some non-markdown files may have issues
|
||||
continue
|
||||
name = filename_to_cname(page['path'])
|
||||
# TODO add email?
|
||||
body = dict(name=name,
|
||||
content=page.data,
|
||||
message=page.info['message'],
|
||||
username=page.info['author'],
|
||||
updated_on=entry['mtime'],
|
||||
created_on=entry['ctime'])
|
||||
search.index_wiki(name, body)
|
||||
# Wiki
|
||||
search.delete_index('wiki')
|
||||
wiki = Wiki(current_app.config['WIKI_PATH'])
|
||||
for entry in wiki.get_index():
|
||||
page = wiki.get_page(entry['name'])
|
||||
if not page:
|
||||
# Some non-markdown files may have issues
|
||||
continue
|
||||
# TODO add email?
|
||||
# TODO I have concens about indexing the commit info from latest revision, see #148
|
||||
info = next(page.history)
|
||||
body = dict(name=page.name,
|
||||
content=page.data,
|
||||
message=info['message'],
|
||||
username=info['author'],
|
||||
updated_on=entry['mtime'],
|
||||
created_on=entry['ctime'])
|
||||
search.index_wiki(page.name, body)
|
||||
|
|
|
|||
|
|
@ -104,47 +104,87 @@ class WikiPage(HookMixin):
|
|||
return data
|
||||
|
||||
@property
|
||||
def info(self):
|
||||
cache_key = self._cache_key('info')
|
||||
cached = cache.get(cache_key)
|
||||
if cached:
|
||||
return cached
|
||||
|
||||
info = self.get_history(limit=1)[0]
|
||||
cache.set(cache_key, info)
|
||||
return info
|
||||
|
||||
def get_history(self, limit=100):
|
||||
def history(self):
|
||||
"""Get page history.
|
||||
|
||||
:param limit: Limit history size.
|
||||
:return: list -- List of dicts
|
||||
History can take a long time to generate for repositories with many commits.
|
||||
This returns an iterator to avoid having to load them all at once, and caches
|
||||
as it goes.
|
||||
|
||||
:return: iter -- Iterator over dicts
|
||||
|
||||
"""
|
||||
versions = []
|
||||
|
||||
try:
|
||||
walker = self.wiki.repo.get_walker(paths=[self.filename], max_entries=limit, follow=True)
|
||||
except KeyError:
|
||||
# We don't have a head, no commits
|
||||
return []
|
||||
cache_head = []
|
||||
cache_tail = cache.get(self._cache_key('history')) or [{'_cache_missing': True}]
|
||||
while True:
|
||||
if not cache_tail:
|
||||
return
|
||||
for index, cached_rev in enumerate(cache_tail):
|
||||
if cached_rev.get("_cache_missing"):
|
||||
break
|
||||
else:
|
||||
yield cached_rev
|
||||
cache_head.extend(cache_tail[:index])
|
||||
cache_tail = cache_tail[index+1:]
|
||||
start_sha = cached_rev.get('sha')
|
||||
end_sha = cache_tail[0].get('sha') if cache_tail else None
|
||||
for rev in self._iter_revs(start_sha=start_sha, end_sha=end_sha, filename=cached_rev.get('filename')):
|
||||
cache_head.append(rev)
|
||||
placeholder = {
|
||||
'_cache_missing': True,
|
||||
'sha': rev['sha'],
|
||||
'filename': rev['new_filename']
|
||||
}
|
||||
cache.set(self._cache_key('history'), cache_head + [placeholder] + cache_tail)
|
||||
yield rev
|
||||
cache.set(self._cache_key('history'), cache_head + cache_tail)
|
||||
|
||||
def _iter_revs(self, start_sha=None, end_sha=None, filename=None):
|
||||
if end_sha:
|
||||
end_sha = [end_sha]
|
||||
if not len(self.wiki.repo.open_index()):
|
||||
# Index is empty, no commits
|
||||
return
|
||||
filename = filename or self.filename
|
||||
walker = iter(self.wiki.repo.get_walker(paths=[filename],
|
||||
include=start_sha,
|
||||
exclude=end_sha,
|
||||
follow=True))
|
||||
if start_sha:
|
||||
# If we are not starting from HEAD, we already have the start commit
|
||||
next(walker)
|
||||
filename = self.filename
|
||||
for entry in walker:
|
||||
change_type = None
|
||||
for change in entry.changes():
|
||||
# Changes should already be filtered to only the one affecting our file
|
||||
change_type = change.type
|
||||
break
|
||||
author_name, author_email = entry.commit.author.rstrip('>').split('<')
|
||||
versions.append(dict(
|
||||
author=author_name.strip(),
|
||||
author_email=author_email,
|
||||
time=entry.commit.author_time,
|
||||
message=entry.commit.message,
|
||||
sha=entry.commit.id,
|
||||
type=change_type))
|
||||
if change.new.path == filename:
|
||||
filename = change.old.path
|
||||
change_type = change.type
|
||||
break
|
||||
|
||||
return versions
|
||||
author_name, author_email = entry.commit.author.rstrip('>').split('<')
|
||||
r = dict(author=author_name.strip(),
|
||||
author_email=author_email,
|
||||
time=entry.commit.author_time,
|
||||
message=entry.commit.message,
|
||||
sha=entry.commit.id,
|
||||
type=change_type,
|
||||
new_filename=change.new.path,
|
||||
old_filename=change.old.path)
|
||||
yield r
|
||||
|
||||
@property
|
||||
def history_cache(self):
|
||||
"""Get info about the history cache.
|
||||
|
||||
:return: tuple -- (cached items, cache complete?)
|
||||
"""
|
||||
cached_revs = cache.get(self._cache_key('history'))
|
||||
if not cached_revs:
|
||||
return 0, False
|
||||
elif any(rev.get('_cache_missing') for rev in cached_revs):
|
||||
return len(cached_revs) - 1, False
|
||||
return len(cached_revs), True
|
||||
|
||||
@property
|
||||
def partials(self):
|
||||
|
|
@ -191,8 +231,14 @@ class WikiPage(HookMixin):
|
|||
|
||||
return username, email
|
||||
|
||||
def _clear_cache(self):
|
||||
cache.delete_many(*(self._cache_key(p) for p in ['data', 'info']))
|
||||
def _invalidate_cache(self, save_history=None):
|
||||
cache.delete(self._cache_key('data'))
|
||||
if save_history:
|
||||
if not save_history[0].get('_cache_missing'):
|
||||
save_history = [{'_cache_missing': True}] + save_history
|
||||
cache.set(self._cache_key('history'), save_history)
|
||||
else:
|
||||
cache.delete(self._cache_key('history'))
|
||||
|
||||
def delete(self, username=None, email=None, message=None):
|
||||
"""Delete page.
|
||||
|
|
@ -211,7 +257,7 @@ class WikiPage(HookMixin):
|
|||
email=email,
|
||||
message=message,
|
||||
files=[self.filename])
|
||||
self._clear_cache()
|
||||
self._invalidate_cache()
|
||||
return commit
|
||||
|
||||
def rename(self, new_name, username=None, email=None, message=None):
|
||||
|
|
@ -245,11 +291,12 @@ class WikiPage(HookMixin):
|
|||
message=message,
|
||||
files=[old_filename, new_filename])
|
||||
|
||||
self._clear_cache()
|
||||
old_history = cache.get(self._cache_key('history'))
|
||||
self._invalidate_cache()
|
||||
self.name = new_name
|
||||
self.filename = new_filename
|
||||
# We need to clear the cache for the new name as well as the old
|
||||
self._clear_cache()
|
||||
self._invalidate_cache(save_history=old_history)
|
||||
|
||||
return commit
|
||||
|
||||
|
|
@ -281,7 +328,8 @@ class WikiPage(HookMixin):
|
|||
message=message,
|
||||
files=[self.filename])
|
||||
|
||||
self._clear_cache()
|
||||
old_history = cache.get(self._cache_key('history'))
|
||||
self._invalidate_cache(save_history=old_history)
|
||||
return ret
|
||||
|
||||
def revert(self, commit_sha, message, username, email):
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ class WikiTest(WikiBaseTest):
|
|||
self.assert_200(rv)
|
||||
|
||||
self.assert_context('name', 'test')
|
||||
eq_(self.get_context_variable('page').info['message'], 'test message')
|
||||
eq_(next(self.get_context_variable('page').history)['message'], 'test message')
|
||||
eq_(self.get_context_variable('page').data, 'testing')
|
||||
|
||||
def test_history(self):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import itertools
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from flask import abort, g, render_template, request, redirect, Blueprint, flash, url_for, current_app
|
||||
from flask.ext.login import login_required, current_user
|
||||
from realms.lib.util import to_canonical, remove_ext, gravatar_url
|
||||
|
|
@ -64,11 +65,37 @@ def revert():
|
|||
def history(name):
|
||||
if current_app.config.get('PRIVATE_WIKI') and current_user.is_anonymous():
|
||||
return current_app.login_manager.unauthorized()
|
||||
return render_template('wiki/history.html', name=name)
|
||||
|
||||
hist = g.current_wiki.get_page(name).get_history()
|
||||
for item in hist:
|
||||
|
||||
@blueprint.route("/_history_data/<path:name>")
|
||||
def history_data(name):
|
||||
"""Ajax provider for paginated history data."""
|
||||
if current_app.config.get('PRIVATE_WIKI') and current_user.is_anonymous():
|
||||
return current_app.login_manager.unauthorized()
|
||||
draw = int(request.args.get('draw', 0))
|
||||
start = int(request.args.get('start', 0))
|
||||
length = int(request.args.get('length', 10))
|
||||
page = g.current_wiki.get_page(name)
|
||||
items = list(itertools.islice(page.history, start, start + length))
|
||||
for item in items:
|
||||
item['gravatar'] = gravatar_url(item['author_email'])
|
||||
return render_template('wiki/history.html', name=name, history=hist)
|
||||
item['DT_RowId'] = item['sha']
|
||||
date = datetime.fromtimestamp(item['time'])
|
||||
item['date'] = date.strftime(current_app.config.get('DATETIME_FORMAT', '%b %d, %Y %I:%M %p'))
|
||||
item['link'] = url_for('.commit', name=name, sha=item['sha'])
|
||||
total_records, hist_complete = page.history_cache
|
||||
if not hist_complete:
|
||||
# Force datatables to fetch more data when it gets to the end
|
||||
total_records += 1
|
||||
return {
|
||||
'draw': draw,
|
||||
'recordsTotal': total_records,
|
||||
'recordsFiltered': total_records,
|
||||
'data': items,
|
||||
'fully_loaded': hist_complete
|
||||
}
|
||||
|
||||
|
||||
|
||||
@blueprint.route("/_edit/<path:name>")
|
||||
|
|
@ -85,7 +112,8 @@ def edit(name):
|
|||
return render_template('wiki/edit.html',
|
||||
name=cname,
|
||||
content=page.data,
|
||||
info=page.info,
|
||||
# TODO: Remove this? See #148
|
||||
info=next(page.history),
|
||||
sha=page.sha,
|
||||
partials=page.partials)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue