Merge branch 'master' into no_gittle

# Conflicts:
#	realms/modules/wiki/models.py

docker/Dockerfile

@@ -28,7 +28,6 @@ ENV GEVENT_RESOLVER=ares
 ENV REALMS_ENV=docker
 ENV REALMS_WIKI_PATH=/home/wiki/data/repo
 ENV REALMS_DB_URI='sqlite:////home/wiki/data/wiki.db'
-ENV REALMS_SQLALCHEMY_DATABASE_URI=${REALMS_DB_URI}
 
 RUN mkdir /home/wiki/data && touch /home/wiki/data/.a
 VOLUME /home/wiki/data

realms/__init__.py

@@ -1,3 +1,4 @@
+import functools
 import sys
 
 # Set default encoding to UTF-8
@@ -75,7 +76,9 @@ class Application(Flask):
 
             # Click
             if hasattr(sources, 'commands'):
-                cli.add_command(sources.commands.cli, name=module_name)
+                if sources.commands.cli.name == 'cli':
+                    sources.commands.cli.name = module_name
+                cli.add_command(sources.commands.cli)
 
             # Hooks
             if hasattr(sources, 'hooks'):
@@ -287,9 +290,8 @@ class AppGroup(click.Group):
         kwargs.setdefault('cls', AppGroup)
         return click.Group.group(self, *args, **kwargs)
 
-flask_cli = AppGroup()
+cli = AppGroup()
 
+# Decorator to be used in modules instead of click.group
+cli_group = functools.partial(click.group, cls=AppGroup)
 
-@flask_cli.group()
-def cli():
-    pass

realms/commands.py

@@ -1,4 +1,4 @@
-from realms import config, create_app, db, __version__, flask_cli as cli, cache
+from realms import config, create_app, db, __version__, cli, cache
 from realms.lib.util import random_string, in_virtualenv, green, yellow, red
 from subprocess import call, Popen
 from multiprocessing import cpu_count

realms/config/__init__.py

@@ -23,7 +23,7 @@ class Config(object):
     BASE_URL = 'http://localhost'
     SITE_TITLE = "Realms"
 
-    # https://pythonhosted.org/Flask-SQLAlchemy/config.html#connection-uri-format
+    # http://flask-sqlalchemy.pocoo.org/config/#connection-uri-format
     DB_URI = 'sqlite:////tmp/wiki.db'
     # DB_URI = 'mysql://scott:tiger@localhost/mydatabase'
     # DB_URI = 'postgresql://scott:tiger@localhost/mydatabase'

realms/lib/hook.py

@@ -25,9 +25,15 @@ class HookMixinMeta(type):
     def __new__(cls, name, bases, attrs):
         super_new = super(HookMixinMeta, cls).__new__
 
+        hookable = []
         for key, value in attrs.items():
+            # Disallow hooking methods which start with an underscore (allow __init__ etc. still)
+            if key.startswith('_') and not key.startswith('__'):
+                continue
             if callable(value):
                 attrs[key] = hook_func(key, value)
+                hookable.append(key)
+        attrs['_hookable'] = hookable
 
         return super_new(cls, name, bases, attrs)
 
@@ -37,9 +43,12 @@ class HookMixin(object):
 
     _pre_hooks = {}
     _post_hooks = {}
+    _hookable = []
 
     @classmethod
     def after(cls, method_name):
+        assert method_name in cls._hookable, "'%s' not a hookable method of '%s'" % (method_name, cls.__name__)
+
         def outer(f, *args, **kwargs):
             cls._post_hooks.setdefault(method_name, []).append((f, args, kwargs))
             return f
@@ -47,6 +56,8 @@ class HookMixin(object):
 
     @classmethod
     def before(cls, method_name):
+        assert method_name in cls._hookable, "'%s' not a hookable method of '%s'" % (method_name, cls.__name__)
+
         def outer(f, *args, **kwargs):
             cls._pre_hooks.setdefault(method_name, []).append((f, args, kwargs))
             return f

realms/modules/auth/local/commands.py

@@ -2,10 +2,10 @@ import click
 from realms.lib.util import random_string
 from realms.modules.auth.local.models import User
 from realms.lib.util import green, red, yellow
-from realms import flask_cli
+from realms import cli_group
 
 
-@flask_cli.group(short_help="Auth Module")
+@cli_group(short_help="Auth Module")
 def cli():
     pass
 

realms/modules/auth/oauth/models.py

@@ -40,7 +40,7 @@ providers = {
         'field_map': {
             'id': 'id',
             'username': 'login',
-            'email': 'email'
+            'email': lambda(data): data.get('email') or data['login'] + '@users.noreply.github.com'
         },
         'token_name': 'access_token'
     },
@@ -118,6 +118,8 @@ class User(BaseUser):
         def get_value(d, key):
             if isinstance(key, basestring):
                 return d.get(key)
+            elif callable(key):
+                return key(d)
             # key should be list here
             val = d.get(key.pop(0))
             if len(key) == 0:

realms/modules/search/commands.py

@@ -1,10 +1,10 @@
 import click
-from realms import create_app, search, flask_cli
+from flask import current_app
+from realms import search, cli_group
 from realms.modules.wiki.models import Wiki
-from realms.lib.util import filename_to_cname
 
 
-@flask_cli.group(short_help="Search Module")
+@cli_group(short_help="Search Module")
 def cli():
     pass
 
@@ -13,27 +13,25 @@ def cli():
 def rebuild_index():
     """ Rebuild search index
     """
-    app = create_app()
-
-    if app.config.get('SEARCH_TYPE') == 'simple':
+    if current_app.config.get('SEARCH_TYPE') == 'simple':
         click.echo("Search type is simple, try using elasticsearch.")
         return
 
-    with app.app_context():
-        # Wiki
-        search.delete_index('wiki')
-        wiki = Wiki(app.config['WIKI_PATH'])
-        for entry in wiki.get_index():
-            page = wiki.get_page(entry['name'])
-            if not page:
-                # Some non-markdown files may have issues
-                continue
-            name = filename_to_cname(page['path'])
-            # TODO add email?
-            body = dict(name=name,
-                        content=page.data,
-                        message=page.info['message'],
-                        username=page.info['author'],
-                        updated_on=entry['mtime'],
-                        created_on=entry['ctime'])
-            search.index_wiki(name, body)
+    # Wiki
+    search.delete_index('wiki')
+    wiki = Wiki(current_app.config['WIKI_PATH'])
+    for entry in wiki.get_index():
+        page = wiki.get_page(entry['name'])
+        if not page:
+            # Some non-markdown files may have issues
+            continue
+        # TODO add email?
+        # TODO I have concens about indexing the commit info from latest revision, see #148
+        info = next(page.history)
+        body = dict(name=page.name,
+                    content=page.data,
+                    message=info['message'],
+                    username=info['author'],
+                    updated_on=entry['mtime'],
+                    created_on=entry['ctime'])
+        search.index_wiki(page.name, body)

realms/modules/wiki/models.py

@@ -104,47 +104,87 @@ class WikiPage(HookMixin):
         return data
 
     @property
-    def info(self):
-        cache_key = self._cache_key('info')
-        cached = cache.get(cache_key)
-        if cached:
-            return cached
-
-        info = self.get_history(limit=1)[0]
-        cache.set(cache_key, info)
-        return info
-
-    def get_history(self, limit=100):
+    def history(self):
         """Get page history.
 
-        :param limit: Limit history size.
-        :return: list -- List of dicts
+        History can take a long time to generate for repositories with many commits.
+        This returns an iterator to avoid having to load them all at once, and caches
+        as it goes.
+
+        :return: iter -- Iterator over dicts
 
         """
-        versions = []
-
-        try:
-            walker = self.wiki.repo.get_walker(paths=[self.filename], max_entries=limit, follow=True)
-        except KeyError:
-            # We don't have a head, no commits
-            return []
+        cache_head = []
+        cache_tail = cache.get(self._cache_key('history')) or [{'_cache_missing': True}]
+        while True:
+            if not cache_tail:
+                return
+            for index, cached_rev in enumerate(cache_tail):
+                if cached_rev.get("_cache_missing"):
+                    break
+                else:
+                    yield cached_rev
+            cache_head.extend(cache_tail[:index])
+            cache_tail = cache_tail[index+1:]
+            start_sha = cached_rev.get('sha')
+            end_sha = cache_tail[0].get('sha') if cache_tail else None
+            for rev in self._iter_revs(start_sha=start_sha, end_sha=end_sha, filename=cached_rev.get('filename')):
+                cache_head.append(rev)
+                placeholder = {
+                    '_cache_missing': True,
+                    'sha': rev['sha'],
+                    'filename': rev['new_filename']
+                }
+                cache.set(self._cache_key('history'), cache_head + [placeholder] + cache_tail)
+                yield rev
+            cache.set(self._cache_key('history'), cache_head + cache_tail)
 
+    def _iter_revs(self, start_sha=None, end_sha=None, filename=None):
+        if end_sha:
+            end_sha = [end_sha]
+        if not len(self.wiki.repo.open_index()):
+            # Index is empty, no commits
+            return
+        filename = filename or self.filename
+        walker = iter(self.wiki.repo.get_walker(paths=[filename],
+                                                include=start_sha,
+                                                exclude=end_sha,
+                                                follow=True))
+        if start_sha:
+            # If we are not starting from HEAD, we already have the start commit
+            next(walker)
+        filename = self.filename
         for entry in walker:
             change_type = None
             for change in entry.changes():
-                # Changes should already be filtered to only the one affecting our file
-                change_type = change.type
-                break
-            author_name, author_email = entry.commit.author.rstrip('>').split('<')
-            versions.append(dict(
-                author=author_name.strip(),
-                author_email=author_email,
-                time=entry.commit.author_time,
-                message=entry.commit.message,
-                sha=entry.commit.id,
-                type=change_type))
+                if change.new.path == filename:
+                    filename = change.old.path
+                    change_type = change.type
+                    break
 
-        return versions
+            author_name, author_email = entry.commit.author.rstrip('>').split('<')
+            r = dict(author=author_name.strip(),
+                     author_email=author_email,
+                     time=entry.commit.author_time,
+                     message=entry.commit.message,
+                     sha=entry.commit.id,
+                     type=change_type,
+                     new_filename=change.new.path,
+                     old_filename=change.old.path)
+            yield r
+
+    @property
+    def history_cache(self):
+        """Get info about the history cache.
+
+        :return: tuple -- (cached items, cache complete?)
+        """
+        cached_revs = cache.get(self._cache_key('history'))
+        if not cached_revs:
+            return 0, False
+        elif any(rev.get('_cache_missing') for rev in cached_revs):
+            return len(cached_revs) - 1, False
+        return len(cached_revs), True
 
     @property
     def partials(self):
@@ -191,8 +231,14 @@ class WikiPage(HookMixin):
 
         return username, email
 
-    def _clear_cache(self):
-        cache.delete_many(*(self._cache_key(p) for p in ['data', 'info']))
+    def _invalidate_cache(self, save_history=None):
+        cache.delete(self._cache_key('data'))
+        if save_history:
+            if not save_history[0].get('_cache_missing'):
+                save_history = [{'_cache_missing': True}] + save_history
+            cache.set(self._cache_key('history'), save_history)
+        else:
+            cache.delete(self._cache_key('history'))
 
     def delete(self, username=None, email=None, message=None):
         """Delete page.
@@ -211,7 +257,7 @@ class WikiPage(HookMixin):
                                   email=email,
                                   message=message,
                                   files=[self.filename])
-        self._clear_cache()
+        self._invalidate_cache()
         return commit
 
     def rename(self, new_name, username=None, email=None, message=None):
@@ -245,11 +291,12 @@ class WikiPage(HookMixin):
                                   message=message,
                                   files=[old_filename, new_filename])
 
-        self._clear_cache()
+        old_history = cache.get(self._cache_key('history'))
+        self._invalidate_cache()
         self.name = new_name
         self.filename = new_filename
         # We need to clear the cache for the new name as well as the old
-        self._clear_cache()
+        self._invalidate_cache(save_history=old_history)
 
         return commit
 
@@ -281,7 +328,8 @@ class WikiPage(HookMixin):
                                message=message,
                                files=[self.filename])
 
-        self._clear_cache()
+        old_history = cache.get(self._cache_key('history'))
+        self._invalidate_cache(save_history=old_history)
         return ret
 
     def revert(self, commit_sha, message, username, email):

realms/modules/wiki/tests.py

@@ -41,7 +41,7 @@ class WikiTest(WikiBaseTest):
         self.assert_200(rv)
 
         self.assert_context('name', 'test')
-        eq_(self.get_context_variable('page').info['message'], 'test message')
+        eq_(next(self.get_context_variable('page').history)['message'], 'test message')
         eq_(self.get_context_variable('page').data, 'testing')
 
     def test_history(self):

realms/modules/wiki/views.py

@@ -1,5 +1,6 @@
 import itertools
 import sys
+from datetime import datetime
 from flask import abort, g, render_template, request, redirect, Blueprint, flash, url_for, current_app
 from flask.ext.login import login_required, current_user
 from realms.lib.util import to_canonical, remove_ext, gravatar_url
@@ -64,11 +65,37 @@ def revert():
 def history(name):
     if current_app.config.get('PRIVATE_WIKI') and current_user.is_anonymous():
         return current_app.login_manager.unauthorized()
+    return render_template('wiki/history.html', name=name)
 
-    hist = g.current_wiki.get_page(name).get_history()
-    for item in hist:
+
+@blueprint.route("/_history_data/<path:name>")
+def history_data(name):
+    """Ajax provider for paginated history data."""
+    if current_app.config.get('PRIVATE_WIKI') and current_user.is_anonymous():
+        return current_app.login_manager.unauthorized()
+    draw = int(request.args.get('draw', 0))
+    start = int(request.args.get('start', 0))
+    length = int(request.args.get('length', 10))
+    page = g.current_wiki.get_page(name)
+    items = list(itertools.islice(page.history, start, start + length))
+    for item in items:
         item['gravatar'] = gravatar_url(item['author_email'])
-    return render_template('wiki/history.html', name=name, history=hist)
+        item['DT_RowId'] = item['sha']
+        date = datetime.fromtimestamp(item['time'])
+        item['date'] = date.strftime(current_app.config.get('DATETIME_FORMAT', '%b %d, %Y %I:%M %p'))
+        item['link'] = url_for('.commit', name=name, sha=item['sha'])
+    total_records, hist_complete = page.history_cache
+    if not hist_complete:
+        # Force datatables to fetch more data when it gets to the end
+        total_records += 1
+    return {
+        'draw': draw,
+        'recordsTotal': total_records,
+        'recordsFiltered': total_records,
+        'data': items,
+        'fully_loaded': hist_complete
+    }
+
 
 
 @blueprint.route("/_edit/<path:name>")
@@ -85,7 +112,8 @@ def edit(name):
     return render_template('wiki/edit.html',
                            name=cname,
                            content=page.data,
-                           info=page.info,
+                           # TODO: Remove this? See #148
+                           info=next(page.history),
                            sha=page.sha,
                            partials=page.partials)
 

realms/templates/errors/404.html

@@ -2,5 +2,8 @@
 {% block body %}
 
   <h1>Page Not Found</h1>
+  {% if error is defined %}
+  <h4>{{ error.description }}</h4>
+  {% endif %}
 
-{% endblock %}
+{% endblock %}

realms/templates/wiki/history.html

@@ -6,27 +6,17 @@
     <a class="btn btn-default btn-sm compare-revisions">Compare Revisions</a>
   </p>
 
-  <table class="table table-bordered revision-tbl">
+  <table class="table table-bordered revision-tbl dataTable DTTT_selectable">
     <thead>
     <tr>
-      <th></th>
       <th>Name</th>
       <th>Revision Message</th>
       <th>Date</th>
     </tr>
     </thead>
-    {% for h in history %}
-      <tr>
-        <td class="checkbox-cell text-center">
-          {% if h.type != 'delete' %}
-          <input type="checkbox" name="versions[]" value="{{ h.sha }}" />
-          {% endif %}
-        </td>
-        <td><img src="{{ h.gravatar }}?s=20" class="avatar"/> {{ h.author }}</td>
-        <td><a href="{{ url_for('wiki.commit', name=name, sha=h.sha) }}" class='label label-primary'>View</a> {{ h.message }} </td>
-        <td>{{ h.time|datetime }}</td>
-      </tr>
-    {% endfor %}
+    <tbody>
+    <tr><td colspan="3" style="text-align: center">Loading file history...</td></tr>
+    </tbody>
   </table>
   <p>
     <a class="btn btn-default btn-sm compare-revisions">Compare Revisions</a>
@@ -34,25 +24,95 @@
 
 {% endblock %}
 
+{% block css %}
+  <style type="text/css">
+    table.dataTable td {
+      transition: background-color 0.5s linear, color 0.5s linear;
+      transition-delay: 0.1s;
+    }
+    table.dataTable tr.active td {
+      transition: background-color 0.1s linear, color 0.1s linear;
+      transition-delay: 0s
+    }
+    table.dataTable tbody tr:hover {
+      background-color: #d8d8d8 !important;
+    }
+  </style>
+{% endblock %}
+
 {% block js %}
   <script>
-    $(function(){
-      $('.revision-tbl :checkbox').change(function () {
-        var $cs=$(this).closest('.revision-tbl').find(':checkbox:checked');
-        if ($cs.length > 2) {
-          this.checked=false;
+    $(document).ready(function() {
+      var selected = [];
+      var selected_pos = [];
+
+      $('.dataTable').dataTable({
+        serverSide: true,
+        ajax: {
+          url: '{{ url_for('.history_data', name=name) }}',
+          dataSrc: function (data) {
+            $('.dataTable').data('fully_loaded', data.fully_loaded);
+            return data.data
+          }
+        },
+        ordering: false,
+        bFilter: false,
+        columns: [
+          {
+            "data": null,
+            "render": function (data) {
+              return '<img src="' + data.gravatar + '?s=20" class="avatar" />&nbsp;&nbsp;' + data.author
+            }
+          },
+          {
+            "data": null,
+            "render": function (data) {
+              return '<a href="' + data.link + '" class="label label-primary">View</a>&nbsp;&nbsp;' + data.message
+            }
+          },
+          { "data": "date" }
+        ],
+        rowCallback: function( row, data, index ) {
+            index += $('.dataTable').DataTable().page.info().start;
+            $(row).data('index', index);
+            if ( $.inArray(data.DT_RowId, selected) !== -1 ) {
+                $(row).addClass('active');
+            }
+        },
+        infoCallback: function( settings, start, end, max, total, pre ) {
+          if (!$('.dataTable').data('fully_loaded')) {
+            total += "+"
+          }
+          return "Showing " + start +" to "+ end + " of " + total + " revisions.";
         }
       });
 
+      $('.dataTable tbody').on('click', 'tr', function () {
+        var id = this.id;
+        var selected_index = $.inArray(id, selected);
+
+        if ( selected_index === -1 ) {
+            selected.push( id );
+            selected_pos.push( $(this).data('index') );
+            if ( selected.length > 2) {
+                // Only 2 selected at once
+                var shifted = selected.shift();
+                selected_pos.shift();
+                $('#' + shifted).removeClass('active');
+            }
+        } else {
+            selected.splice( selected_index, 1 );
+            selected_pos.splice( selected_index, 1);
+        }
+
+        $(this).toggleClass('active');
+      });
       $(".compare-revisions").click(function(){
-        var $cs = $('.revision-tbl').find(':checkbox:checked');
-        if ($cs.length != 2) return;
-        var revs = [];
-        $.each($cs, function(i, v){
-          revs.push(v.value);
-        });
-        revs.reverse();
-        revs = revs.join("..");
+        if (selected.length != 2) return;
+        if (selected_pos[1] > selected_pos[0]) {
+          selected.reverse()
+        }
+        revs = selected.join("..");
         location.href = "{{ config.RELATIVE_PATH }}/_compare/{{ name }}/" + revs;
       });
     });

realms/version.py

@@ -1 +1 @@
-__version__ = '0.8.0'
+__version__ = '0.8.1'