Personal/realms-wiki
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

cli and setuptools wip

Browse source
This commit is contained in:
Matthew Scragg 2014-10-16 16:54:45 -05:00
parent 142050d804
commit 07852bdd98
17 changed files with 496 additions and 242 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.gitignore vendored
View file

@ -1,12 +1,14 @@
.vagrant
.venv
.venv-pypy
.idea
.webassets-cache
*.pyc
*.egg-info
*.pid
dist
build
pidfile
/dist
/build
config.py
config.sls
config.json

2
.travis.yml
View file

@ -5,6 +5,6 @@ python:
before_install:
- sudo apt-get install -y libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libyaml-dev libssl-dev
install: "pip install -r dev-requirements.txt"
install: "pip install -r requirements-dev.txt"
script: nosetests

6
MANIFEST.in Normal file
View file

@ -0,0 +1,6 @@
include requirements.txt VERSION LICENSE
recursive-include realms/static/css *
recursive-include realms/static/fonts *
recursive-include realms/static/js *
recursive-include realms/static/vendor *
recursive-include realms/templates *

0
README.md → README
View file

2
VERSION
View file

@ -1 +1 @@
0.3.1
0.3.13

2
Vagrantfile vendored
View file

@ -6,7 +6,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.provider :virtualbox do |vb|
vb.name = "realms-wiki"
vb.memory = 2048
vb.cpus = 2
vb.cpus = 4
end
config.vm.provision "shell", path: "install.sh"

34
install.sh
View file

@ -50,23 +50,51 @@ sudo -iu ${APP_USER} bower --allow-root --config.cwd=${APP_DIR} --config.directo
sudo -iu ${APP_USER} virtualenv ${APP_DIR}/.venv
sudo -iu ${APP_USER} ${APP_DIR}/.venv/bin/pip install ${APP_DIR}
cd ${APP_DIR} && sudo -iu ${APP_USER} ${APP_DIR}/.venv/bin/pip install -r ${APP_DIR}/requirements-dev.txt
echo "Installing start scripts"
cat << EOF > /usr/local/bin/realms-wiki
#!/bin/bash
${APP_DIR}/.venv/bin/python ${APP_DIR}/manage.py "\$@"
${APP_DIR}/realms-wiki "\$@"
EOF
sudo chmod +x /usr/local/bin/realms-wiki
cat << EOF > /etc/init/realms-wiki.conf
limit nofile 65335 65335
respawn
description "Realms Wiki"
author "scragg@gmail.com"
chdir ${APP_DIR}
env PATH=${APP_DIR}/.venv/bin:/usr/local/bin:/usr/bin:/bin:$PATH
env LC_ALL=en_US.UTF-8
env GEVENT_RESOLVER=ares
export PATH
export LC_ALL
export GEVENT_RESOLVER
setuid ${APP_USER}
setgid ${APP_USER}
start on runlevel [2345]
stop on runlevel [!2345]
respawn
exec /usr/local/bin/realms-wiki run
exec gunicorn \
--name realms-wiki \
--access-logfile - \
--error-logfile - \
--worker-class gevent \
--workers 2 \
--bind 0.0.0.0:5000 \
--user ${APP_USER} \
--group ${APP_USER} \
--chdir ${APP_DIR} \
wsgi:app
EOF

195
manage.py
View file

@ -1,195 +0,0 @@
from gevent import wsgi
from realms import config, app, cli, db
from realms.lib.util import random_string
from subprocess import call
import click
import json
@cli.command()
@click.option('--site-title',
default=config.SITE_TITLE,
prompt='Enter site title.')
@click.option('--base_url',
default=config.BASE_URL,
prompt='Enter base URL.')
@click.option('--port',
default=config.PORT,
prompt='Enter port number.')
@click.option('--secret-key',
default=config.SECRET_KEY if config.SECRET_KEY != "CHANGE_ME" else random_string(64),
prompt='Enter secret key.')
@click.option('--wiki-path',
default=config.WIKI_PATH,
prompt='Where do you want to store wiki data?',
help='Wiki Directory (git repo)')
@click.option('--allow-anon',
default=config.ALLOW_ANON,
is_flag=True,
prompt='Allow anonymous edits?')
@click.option('--registration-enabled',
default=config.REGISTRATION_ENABLED,
is_flag=True,
prompt='Enable registration?')
@click.option('--cache-type',
default=config.CACHE_TYPE,
type=click.Choice([None, 'simple', 'redis', 'memcached']),
prompt='Cache type?')
@click.option('--db-uri',
default=config.DB_URI,
prompt='Database URI, Examples: http://goo.gl/RyW0cl')
@click.pass_context
def setup(ctx, **kw):
""" Start setup wizard
"""
conf = {}
for k, v in kw.items():
conf[k.upper()] = v
config.update(conf)
if conf['CACHE_TYPE'] == 'redis':
ctx.invoke(setup_redis)
elif conf['CACHE_TYPE'] == 'memcached':
ctx.invoke(setup_memcached)
click.secho('Config saved to %s/config.json' % config.APP_PATH, fg='green')
click.secho('Type "realms-wiki run" to start server', fg='yellow')
@click.command()
@click.option('--cache-redis-host',
default=getattr(config, 'CACHE_REDIS_HOST', "127.0.0.1"),
prompt='Redis host')
@click.option('--cache-redis-port',
default=getattr(config, 'CACHE_REDIS_POST', 6379),
prompt='Redis port')
@click.option('--cache-redis-password',
default=getattr(config, 'CACHE_REDIS_PASSWORD', None),
prompt='Redis password')
@click.option('--cache-redis-db',
default=getattr(config, 'CACHE_REDIS_DB', 0),
prompt='Redis db')
def setup_redis(**kw):
conf = {}
for k, v in kw.items():
conf[k.upper()] = v
config.update(conf)
install_redis()
def get_prefix():
import sys
return sys.prefix
def get_pip():
""" Get virtualenv path for pip
"""
return get_prefix() + '/bin/pip'
@cli.command()
@click.argument('cmd', nargs=-1)
def pip(cmd):
""" Execute pip commands for this virtualenv
"""
call(get_pip() + ' ' + ' '.join(cmd), shell=True)
def install_redis():
call([get_pip(), 'install', 'redis'])
def install_mysql():
call([get_pip(), 'install', 'MySQL-Python'])
def install_postgres():
call([get_pip(), 'install', 'psycopg2'])
def install_memcached():
call([get_pip(), 'install', 'python-memcached'])
@click.command()
@click.option('--cache-memcached-servers',
default=getattr(config, 'CACHE_MEMCACHED_SERVERS', ["127.0.0.1:11211"]),
type=click.STRING,
prompt='Memcached servers, separate with a space')
def setup_memcached(**kw):
conf = {}
for k, v in kw.items():
conf[k.upper()] = v
config.update(conf)
@cli.command()
@click.argument('config_json')
def configure(config_json):
""" Set config.json, expects JSON encoded string
"""
try:
config.update(json.loads(config_json))
except ValueError, e:
click.secho('Config value should be valid JSON', fg='red')
@cli.command()
@click.option('--port', default=5000)
def dev(port):
""" Run development server
"""
click.secho("Starting development server", fg='green')
app.run(host="0.0.0.0",
port=port,
debug=True)
@cli.command()
def run():
""" Run production server
"""
click.secho("Server started. Env: %s Port: %s" % (config.ENV, config.PORT), fg='green')
wsgi.WSGIServer(('', int(config.PORT)), app).serve_forever()
@cli.command()
def create_db():
""" Creates DB tables
"""
click.echo("Creating all tables")
db.create_all()
@cli.command()
@click.confirmation_option(help='Are you sure you want to drop the db?')
def drop_db():
""" Drops DB tables
"""
click.echo("Dropping all tables")
db.drop_all()
@cli.command()
def test():
""" Run tests
"""
call([get_prefix() + "/bin/nosetests", config.APP_PATH])
@cli.command()
def version():
""" Output version
"""
with open('VERSION') as f:
return f.read().strip()
if __name__ == '__main__':
cli()

6
realms-wiki Executable file
View file

@ -0,0 +1,6 @@
#!/usr/bin/env python
from realms.cli import cli
if __name__ == '__main__':
cli()

9
realms/__init__.py
View file

@ -1,10 +1,4 @@
import sys
if 'threading' in sys.modules:
del sys.modules['threading']
# Monkey patch stdlib.
import gevent.monkey
gevent.monkey.patch_all(aggressive=False, subprocess=True)
# Set default encoding to UTF-8
reload(sys)
@ -12,7 +6,6 @@ reload(sys)
sys.setdefaultencoding('utf-8')
import time
import sys
import json
import httplib
import traceback
@ -74,7 +67,7 @@ class Application(Flask):
if hasattr(sources, 'commands'):
cli.add_command(sources.commands.cli, name=module_name)
print >> sys.stderr, ' * Ready in %.2fms' % (1000.0 * (time.time() - start_time))
# print >> sys.stderr, ' * Ready in %.2fms' % (1000.0 * (time.time() - start_time))
def make_response(self, rv):
if rv is None:

350
realms/cli.py Normal file
View file

@ -0,0 +1,350 @@
from realms import config, app, cli, db
from realms.lib.util import random_string
from subprocess import call, Popen
from multiprocessing import cpu_count
import click
import json
import sys
import os
def get_user():
for name in ('SUDO_USER', 'LOGNAME', 'USER', 'LNAME', 'USERNAME'):
user = os.environ.get(name)
if user:
return user
def in_virtualenv():
return hasattr(sys, 'real_prefix')
def is_su():
return os.geteuid() == 0
def get_pid():
try:
with file(config.PIDFILE) as f:
pid = f.read().strip()
return pid if pid and int(pid) > 0 and not call(['kill', '-s', '0', pid]) else False
except IOError:
return False
def module_exists(module_name):
try:
__import__(module_name)
except ImportError:
return False
else:
return True
@cli.command()
@click.option('--site-title',
default=config.SITE_TITLE,
prompt='Enter site title.')
@click.option('--base_url',
default=config.BASE_URL,
prompt='Enter base URL.')
@click.option('--port',
default=config.PORT,
prompt='Enter port number.')
@click.option('--secret-key',
default=config.SECRET_KEY if config.SECRET_KEY != "CHANGE_ME" else random_string(64),
prompt='Enter secret key.')
@click.option('--wiki-path',
default=config.WIKI_PATH,
prompt='Where do you want to store wiki data?',
help='Wiki Directory (git repo)')
@click.option('--allow-anon',
default=config.ALLOW_ANON,
is_flag=True,
prompt='Allow anonymous edits?')
@click.option('--registration-enabled',
default=config.REGISTRATION_ENABLED,
is_flag=True,
prompt='Enable registration?')
@click.option('--cache-type',
default=config.CACHE_TYPE,
type=click.Choice([None, 'simple', 'redis', 'memcached']),
prompt='Cache type?')
@click.option('--db-uri',
default=config.DB_URI,
prompt='Database URI? Examples: http://goo.gl/RyW0cl')
@click.pass_context
def setup(ctx, **kw):
""" Start setup wizard
"""
if not in_virtualenv() and not is_su():
# This does not account for people the have user level python installs
# that aren't virtual environments! Should be rare I think
click.secho("Setup requires root privileges, use sudo or run as root")
return
conf = {}
for k, v in kw.items():
conf[k.upper()] = v
conf_path = config.update(conf)
if conf['CACHE_TYPE'] == 'redis':
ctx.invoke(setup_redis)
elif conf['CACHE_TYPE'] == 'memcached':
ctx.invoke(setup_memcached)
click.secho('Config saved to %s' % conf_path, fg='green')
click.secho('Type "realms-wiki start" to start server', fg='yellow')
click.secho('Type "realms-wiki dev" to start server in development mode', fg='yellow')
@click.command()
@click.option('--cache-redis-host',
default=getattr(config, 'CACHE_REDIS_HOST', "127.0.0.1"),
prompt='Redis host')
@click.option('--cache-redis-port',
default=getattr(config, 'CACHE_REDIS_POST', 6379),
prompt='Redis port')
@click.option('--cache-redis-password',
default=getattr(config, 'CACHE_REDIS_PASSWORD', None),
prompt='Redis password')
@click.option('--cache-redis-db',
default=getattr(config, 'CACHE_REDIS_DB', 0),
prompt='Redis db')
def setup_redis(**kw):
conf = {}
for k, v in kw.items():
conf[k.upper()] = v
config.update(conf)
install_redis()
def get_prefix():
return sys.prefix
def get_pip():
""" Get virtualenv path for pip
"""
if not in_virtualenv() and not is_su():
click.secho("This command requires root, use sudo or run as root")
return
if in_virtualenv():
return get_prefix() + '/bin/pip'
else:
return 'pip'
@cli.command()
@click.argument('cmd', nargs=-1)
def pip(cmd):
""" Execute pip commands, useful for virtualenvs
"""
call(get_pip() + ' ' + ' '.join(cmd), shell=True)
def install_redis():
call([get_pip(), 'install', 'redis'])
def install_mysql():
call([get_pip(), 'install', 'MySQL-Python'])
def install_postgres():
call([get_pip(), 'install', 'psycopg2'])
def install_memcached():
call([get_pip(), 'install', 'python-memcached'])
@click.command()
@click.option('--cache-memcached-servers',
default=getattr(config, 'CACHE_MEMCACHED_SERVERS', ["127.0.0.1:11211"]),
type=click.STRING,
prompt='Memcached servers, separate with a space')
def setup_memcached(**kw):
conf = {}
for k, v in kw.items():
conf[k.upper()] = v
config.update(conf)
@cli.command()
@click.option('--user',
default=get_user(),
type=click.STRING,
prompt='Run as which user? (it must exist)')
@click.option('--port',
default=config.PORT,
type=click.INT,
prompt='What port to listen on?')
@click.option('--workers',
default=cpu_count() * 2 + 1,
type=click.INT,
prompt="Number of workers? (defaults to ncpu*2+1)")
def setup_upstart(**kwargs):
""" Start upstart conf creation wizard
"""
from realms.lib.util import upstart_script
import realms
print os.path.dirname(realms.__file__)
if not is_su():
click.secho("Please run this command as root or use sudo", fg='red')
return
if in_virtualenv():
app_dir = get_prefix()
path = '/'.join(sys.executable.split('/')[:-1])
else:
# Assumed root install, not sure if this matters?
app_dir = '/'
path = None
kwargs.update(dict(app_dir=app_dir, path=path))
conf_file = '/etc/init/realms-wiki.conf'
with open('/etc/init/realms-wiki.conf', 'w') as f:
f.write(upstart_script(**kwargs))
click.secho('Wrote file to %s' % conf_file, fg='yellow')
click.echo("Type 'sudo start realms-wiki' to start")
click.echo("Type 'sudo stop realms-wiki' to stop")
click.echo("Type 'sudo restart realms-wiki' to restart")
@cli.command()
@click.argument('json_string')
def configure(json_string):
""" Set config.json, expects JSON encoded string
"""
try:
config.update(json.loads(json_string))
except ValueError, e:
click.secho('Config value should be valid JSON', fg='red')
@cli.command()
@click.option('--port', default=5000)
def dev(port):
""" Run development server
"""
click.secho("Starting development server", fg='green')
app.run(host="0.0.0.0",
port=port,
debug=True)
def start_server():
if get_pid():
click.echo("Server is already running")
return
flags = '--daemon --pid %s' % config.PIDFILE
click.secho("Server started. Port: %s" % config.PORT, fg='green')
Popen('gunicorn realms:app -b 0.0.0.0:%s -k gevent %s' %
(config.PORT, flags), shell=True, executable='/bin/bash')
def stop_server():
pid = get_pid()
if not pid:
click.echo("Server is not running")
else:
click.echo("Shutting down server")
call(['kill', pid])
@cli.command()
def run():
""" Run production server (alias for start)
"""
start_server()
@cli.command()
def start():
""" Run server daemon
"""
start_server()
@cli.command()
def stop():
""" Stop server
"""
stop_server()
@cli.command()
def restart():
""" Restart server
"""
stop_server()
start_server()
@cli.command()
def status():
""" Get server status
"""
pid = get_pid()
if not pid:
click.echo("Server is not running")
else:
click.echo("Server is running PID: %s" % pid)
@cli.command()
def create_db():
""" Creates DB tables
"""
click.echo("Creating all tables")
db.create_all()
@cli.command()
@click.confirmation_option(help='Are you sure you want to drop the db?')
def drop_db():
""" Drops DB tables
"""
click.echo("Dropping all tables")
db.drop_all()
@cli.command()
def test():
""" Run tests
"""
for mod in [('flask.ext.testing', 'Flask-Testing'), ('nose', 'nose')]:
if not module_exists(mod[0]):
call([get_pip(), 'install', mod[1]])
nosetests = get_prefix() + "/bin/nosetests" if in_virtualenv() else "nosetests"
call([nosetests, config.APP_PATH])
@cli.command()
def version():
""" Output version
"""
with open('VERSION') as f:
return f.read().strip()
if __name__ == '__main__':
cli()

59
realms/config/__init__.py
View file

@ -6,28 +6,48 @@ from urlparse import urlparse
def update(data):
conf = read()
conf.update(data)
save(data)
return save(data)
def read():
conf = dict()
try:
with open(os.path.join(APP_PATH, 'config.json')) as f:
conf = json.load(f)
except IOError:
pass
for k, v in os.environ.items():
if k.startswith('REALMS_'):
conf[k[7:]] = v
for loc in os.curdir, os.path.expanduser("~"), "/etc/realms-wiki", os.environ.get("REALMS_WIKI_CONF"):
try:
if not loc:
continue
with open(os.path.join(loc, "realms-wiki.json")) as f:
conf.update(json.load(f))
break
except IOError:
pass
for k in ['APP_PATH', 'USER_HOME']:
if k in conf:
del conf[k]
return conf
def save(conf):
with open(os.path.join(APP_PATH, 'config.json'), 'w') as f:
f.write(json.dumps(conf, sort_keys=True, indent=4, separators=(',', ': ')).strip() + '\n')
for loc in "/etc/realms-wiki", os.path.expanduser("~"), os.curdir:
try:
with open(os.path.join(loc, 'realms-wiki.json'), 'w') as f:
f.write(json.dumps(conf, sort_keys=True, indent=4, separators=(',', ': ')).strip() + '\n')
return os.path.join(loc, 'realms-wiki.json')
except IOError:
pass
APP_PATH = os.path.abspath(os.path.dirname(__file__) + "/../..")
USER_HOME = os.path.abspath(os.path.expanduser("~"))
# Best to change to /var/run
PIDFILE = "/tmp/realms-wiki.pid"
ENV = 'DEV'
DEBUG = True
@ -39,7 +59,7 @@ BASE_URL = 'http://localhost'
SITE_TITLE = "Realms"
# https://pythonhosted.org/Flask-SQLAlchemy/config.html#connection-uri-format
DB_URI = 'sqlite:///%s/wiki.db' % USER_HOME
DB_URI = 'sqlite:////tmp/wiki.db'
# DB_URI = 'mysql://scott:tiger@localhost/mydatabase'
# DB_URI = 'postgresql://scott:tiger@localhost/mydatabase'
# DB_URI = 'oracle://scott:tiger@127.0.0.1:1521/sidname'
@ -67,7 +87,7 @@ RECAPTCHA_OPTIONS = {}
SECRET_KEY = 'CHANGE_ME'
# Path on file system where wiki data will reside
WIKI_PATH = os.path.join(APP_PATH, 'wiki')
WIKI_PATH = '/tmp/wiki'
# Name of page that will act as home
WIKI_HOME = 'home'
@ -78,7 +98,7 @@ REGISTRATION_ENABLED = True
# Used by Flask-Login
LOGIN_DISABLED = ALLOW_ANON
# None, firepad, or togetherjs
# None, firepad, and/or togetherjs
COLLABORATION = 'togetherjs'
# Required for firepad
@ -91,22 +111,7 @@ LOCKED = WIKI_LOCKED_PAGES
ROOT_ENDPOINT = 'wiki.page'
__env = {}
for k, v in os.environ.items():
if k.startswith('REALMS_'):
__env[k[7:]] = v
globals().update(__env)
try:
with open(os.path.join(APP_PATH, 'config.json')) as f:
__settings = json.load(f)
for k in ['APP_PATH', 'USER_HOME']:
if k in __settings:
del __settings[k]
globals().update(__settings)
except IOError:
pass
globals().update(read())
if BASE_URL.endswith('/'):
BASE_URL = BASE_URL[-1]

47
realms/lib/util.py
View file

@ -4,6 +4,7 @@ import hashlib
import json
import string
import random
from jinja2 import Template
class AttrDict(dict):
@ -97,3 +98,49 @@ def to_canonical(s):
def gravatar_url(email):
return "//www.gravatar.com/avatar/" + hashlib.md5(email).hexdigest()
def upstart_script(user='root', app_dir=None, port=5000, workers=2, path=None):
script = """
limit nofile 65335 65335
respawn
description "Realms Wiki"
author "scragg@gmail.com"
chdir {{ app_dir }}
{% if path %}
env PATH={{ path }}:/usr/local/bin:/usr/bin:/bin:$PATH
export PATH
{% endif %}
env LC_ALL=en_US.UTF-8
env GEVENT_RESOLVER=ares
export LC_ALL
export GEVENT_RESOLVER
setuid {{ user }}
setgid {{ user }}
start on runlevel [2345]
stop on runlevel [!2345]
respawn
exec gunicorn \
--name realms-wiki \
--access-logfile - \
--error-logfile - \
--worker-class gevent \
--workers {{ workers }} \
--bind 0.0.0.0:{{ port }} \
--user {{ user }} \
--group {{ user }} \
--chdir {{ app_dir }} \
realms:app
"""
template = Template(script)
return template.render(user=user, app_dir=app_dir, port=port, workers=workers, path=path)

2
realms/modules/wiki/tests.py
View file

@ -15,9 +15,11 @@ class WikiTest(TestCase):
self.assert_200(self.client.get(url_for("wiki.create")))
""" Create a test page first!
for route in ['page', 'edit', 'history', 'index']:
rv = self.client.get(url_for("wiki.%s" % route, name='test'))
self.assert_200(rv, "wiki.%s: %s" % (route, rv.status_code))
"""
def test_write_page(self):
pass

0
dev-requirements.txt → requirements-dev.txt
View file

1
requirements.txt
View file

@ -11,6 +11,7 @@ click==3.3
gevent==1.0.1
ghdiff==0.4
gittle==0.4.0
gunicorn==19.1.1
itsdangerous==0.24
lxml==3.4.0
markdown2==2.3.0

17
setup.py
View file

@ -1,8 +1,12 @@
from setuptools import setup, find_packages
import os
if os.environ.get('USER', '') == 'vagrant':
del os.link
DESCRIPTION = "Simple git based wiki"
with open('README.md') as f:
with open('README') as f:
LONG_DESCRIPTION = f.read()
with open('requirements.txt') as f:
@ -13,15 +17,20 @@ with open('VERSION') as f:
CLASSIFIERS = [
'Intended Audience :: Developers',
'License :: OSI Approved :: GPLv2 License',
'Operating System :: OS Independent',
'License :: OSI Approved :: GNU General Public License v2 (GPLv2)',
'Operating System :: POSIX :: Linux',
'Programming Language :: Python',
'Topic :: Software Development :: Libraries :: Python Modules']
'Topic :: Internet :: WWW/HTTP :: Dynamic Content']
setup(name='realms-wiki',
version=VERSION,
packages=find_packages(),
install_requires=required,
#scripts=['realms-wiki'],
entry_points={
'console_scripts': [
'realms-wiki = realms.cli:cli'
]},
author='Matthew Scragg',
author_email='scragg@gmail.com',
maintainer='Matthew Scragg',