Browse Source

cli and setuptools wip

master
Matthew Scragg 9 years ago
parent
commit
07852bdd98
17 changed files with 496 additions and 242 deletions
  1. +4
    -2
      .gitignore
  2. +1
    -1
      .travis.yml
  3. +6
    -0
      MANIFEST.in
  4. +0
    -0
      README
  5. +1
    -1
      VERSION
  6. +1
    -1
      Vagrantfile
  7. +31
    -3
      install.sh
  8. +0
    -195
      manage.py
  9. +6
    -0
      realms-wiki
  10. +1
    -8
      realms/__init__.py
  11. +350
    -0
      realms/cli.py
  12. +32
    -27
      realms/config/__init__.py
  13. +47
    -0
      realms/lib/util.py
  14. +2
    -0
      realms/modules/wiki/tests.py
  15. +0
    -0
      requirements-dev.txt
  16. +1
    -0
      requirements.txt
  17. +13
    -4
      setup.py

+ 4
- 2
.gitignore View File

@@ -1,12 +1,14 @@
.vagrant
.venv
.venv-pypy
.idea
.webassets-cache
*.pyc
*.egg-info
*.pid
dist
build
pidfile
/dist
/build
config.py
config.sls
config.json


+ 1
- 1
.travis.yml View File

@@ -5,6 +5,6 @@ python:
before_install:
- sudo apt-get install -y libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libyaml-dev libssl-dev

install: "pip install -r dev-requirements.txt"
install: "pip install -r requirements-dev.txt"

script: nosetests

+ 6
- 0
MANIFEST.in View File

@@ -0,0 +1,6 @@
include requirements.txt VERSION LICENSE
recursive-include realms/static/css *
recursive-include realms/static/fonts *
recursive-include realms/static/js *
recursive-include realms/static/vendor *
recursive-include realms/templates *

README.md → README View File


+ 1
- 1
VERSION View File

@@ -1 +1 @@
0.3.1
0.3.13

+ 1
- 1
Vagrantfile View File

@@ -6,7 +6,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.provider :virtualbox do |vb|
vb.name = "realms-wiki"
vb.memory = 2048
vb.cpus = 2
vb.cpus = 4
end

config.vm.provision "shell", path: "install.sh"


+ 31
- 3
install.sh View File

@@ -50,23 +50,51 @@ sudo -iu ${APP_USER} bower --allow-root --config.cwd=${APP_DIR} --config.directo

sudo -iu ${APP_USER} virtualenv ${APP_DIR}/.venv

sudo -iu ${APP_USER} ${APP_DIR}/.venv/bin/pip install ${APP_DIR}
cd ${APP_DIR} && sudo -iu ${APP_USER} ${APP_DIR}/.venv/bin/pip install -r ${APP_DIR}/requirements-dev.txt

echo "Installing start scripts"
cat << EOF > /usr/local/bin/realms-wiki
#!/bin/bash
${APP_DIR}/.venv/bin/python ${APP_DIR}/manage.py "\$@"
${APP_DIR}/realms-wiki "\$@"
EOF

sudo chmod +x /usr/local/bin/realms-wiki

cat << EOF > /etc/init/realms-wiki.conf
limit nofile 65335 65335

respawn

description "Realms Wiki"
author "scragg@gmail.com"

chdir ${APP_DIR}

env PATH=${APP_DIR}/.venv/bin:/usr/local/bin:/usr/bin:/bin:$PATH
env LC_ALL=en_US.UTF-8
env GEVENT_RESOLVER=ares

export PATH
export LC_ALL
export GEVENT_RESOLVER

setuid ${APP_USER}
setgid ${APP_USER}

start on runlevel [2345]
stop on runlevel [!2345]

respawn
exec /usr/local/bin/realms-wiki run

exec gunicorn \
--name realms-wiki \
--access-logfile - \
--error-logfile - \
--worker-class gevent \
--workers 2 \
--bind 0.0.0.0:5000 \
--user ${APP_USER} \
--group ${APP_USER} \
--chdir ${APP_DIR} \
wsgi:app
EOF

+ 0
- 195
manage.py View File

@@ -1,195 +0,0 @@
from gevent import wsgi
from realms import config, app, cli, db
from realms.lib.util import random_string
from subprocess import call
import click
import json


@cli.command()
@click.option('--site-title',
default=config.SITE_TITLE,
prompt='Enter site title.')
@click.option('--base_url',
default=config.BASE_URL,
prompt='Enter base URL.')
@click.option('--port',
default=config.PORT,
prompt='Enter port number.')
@click.option('--secret-key',
default=config.SECRET_KEY if config.SECRET_KEY != "CHANGE_ME" else random_string(64),
prompt='Enter secret key.')
@click.option('--wiki-path',
default=config.WIKI_PATH,
prompt='Where do you want to store wiki data?',
help='Wiki Directory (git repo)')
@click.option('--allow-anon',
default=config.ALLOW_ANON,
is_flag=True,
prompt='Allow anonymous edits?')
@click.option('--registration-enabled',
default=config.REGISTRATION_ENABLED,
is_flag=True,
prompt='Enable registration?')
@click.option('--cache-type',
default=config.CACHE_TYPE,
type=click.Choice([None, 'simple', 'redis', 'memcached']),
prompt='Cache type?')
@click.option('--db-uri',
default=config.DB_URI,
prompt='Database URI, Examples: http://goo.gl/RyW0cl')
@click.pass_context
def setup(ctx, **kw):
""" Start setup wizard
"""
conf = {}

for k, v in kw.items():
conf[k.upper()] = v

config.update(conf)

if conf['CACHE_TYPE'] == 'redis':
ctx.invoke(setup_redis)
elif conf['CACHE_TYPE'] == 'memcached':
ctx.invoke(setup_memcached)

click.secho('Config saved to %s/config.json' % config.APP_PATH, fg='green')
click.secho('Type "realms-wiki run" to start server', fg='yellow')


@click.command()
@click.option('--cache-redis-host',
default=getattr(config, 'CACHE_REDIS_HOST', "127.0.0.1"),
prompt='Redis host')
@click.option('--cache-redis-port',
default=getattr(config, 'CACHE_REDIS_POST', 6379),
prompt='Redis port')
@click.option('--cache-redis-password',
default=getattr(config, 'CACHE_REDIS_PASSWORD', None),
prompt='Redis password')
@click.option('--cache-redis-db',
default=getattr(config, 'CACHE_REDIS_DB', 0),
prompt='Redis db')
def setup_redis(**kw):
conf = {}

for k, v in kw.items():
conf[k.upper()] = v

config.update(conf)
install_redis()


def get_prefix():
import sys
return sys.prefix


def get_pip():
""" Get virtualenv path for pip
"""
return get_prefix() + '/bin/pip'


@cli.command()
@click.argument('cmd', nargs=-1)
def pip(cmd):
""" Execute pip commands for this virtualenv
"""
call(get_pip() + ' ' + ' '.join(cmd), shell=True)


def install_redis():
call([get_pip(), 'install', 'redis'])


def install_mysql():
call([get_pip(), 'install', 'MySQL-Python'])


def install_postgres():
call([get_pip(), 'install', 'psycopg2'])


def install_memcached():
call([get_pip(), 'install', 'python-memcached'])


@click.command()
@click.option('--cache-memcached-servers',
default=getattr(config, 'CACHE_MEMCACHED_SERVERS', ["127.0.0.1:11211"]),
type=click.STRING,
prompt='Memcached servers, separate with a space')
def setup_memcached(**kw):
conf = {}

for k, v in kw.items():
conf[k.upper()] = v

config.update(conf)


@cli.command()
@click.argument('config_json')
def configure(config_json):
""" Set config.json, expects JSON encoded string
"""
try:
config.update(json.loads(config_json))
except ValueError, e:
click.secho('Config value should be valid JSON', fg='red')


@cli.command()
@click.option('--port', default=5000)
def dev(port):
""" Run development server
"""
click.secho("Starting development server", fg='green')
app.run(host="0.0.0.0",
port=port,
debug=True)


@cli.command()
def run():
""" Run production server
"""
click.secho("Server started. Env: %s Port: %s" % (config.ENV, config.PORT), fg='green')
wsgi.WSGIServer(('', int(config.PORT)), app).serve_forever()


@cli.command()
def create_db():
""" Creates DB tables
"""
click.echo("Creating all tables")
db.create_all()


@cli.command()
@click.confirmation_option(help='Are you sure you want to drop the db?')
def drop_db():
""" Drops DB tables
"""
click.echo("Dropping all tables")
db.drop_all()


@cli.command()
def test():
""" Run tests
"""
call([get_prefix() + "/bin/nosetests", config.APP_PATH])


@cli.command()
def version():
""" Output version
"""
with open('VERSION') as f:
return f.read().strip()

if __name__ == '__main__':
cli()

+ 6
- 0
realms-wiki View File

@@ -0,0 +1,6 @@
#!/usr/bin/env python

from realms.cli import cli

if __name__ == '__main__':
cli()

+ 1
- 8
realms/__init__.py View File

@@ -1,10 +1,4 @@
import sys
if 'threading' in sys.modules:
del sys.modules['threading']

# Monkey patch stdlib.
import gevent.monkey
gevent.monkey.patch_all(aggressive=False, subprocess=True)

# Set default encoding to UTF-8
reload(sys)
@@ -12,7 +6,6 @@ reload(sys)
sys.setdefaultencoding('utf-8')

import time
import sys
import json
import httplib
import traceback
@@ -74,7 +67,7 @@ class Application(Flask):
if hasattr(sources, 'commands'):
cli.add_command(sources.commands.cli, name=module_name)

print >> sys.stderr, ' * Ready in %.2fms' % (1000.0 * (time.time() - start_time))
# print >> sys.stderr, ' * Ready in %.2fms' % (1000.0 * (time.time() - start_time))

def make_response(self, rv):
if rv is None:


+ 350
- 0
realms/cli.py View File

@@ -0,0 +1,350 @@
from realms import config, app, cli, db
from realms.lib.util import random_string
from subprocess import call, Popen
from multiprocessing import cpu_count
import click
import json
import sys
import os


def get_user():
for name in ('SUDO_USER', 'LOGNAME', 'USER', 'LNAME', 'USERNAME'):
user = os.environ.get(name)
if user:
return user


def in_virtualenv():
return hasattr(sys, 'real_prefix')


def is_su():
return os.geteuid() == 0


def get_pid():
try:
with file(config.PIDFILE) as f:
pid = f.read().strip()
return pid if pid and int(pid) > 0 and not call(['kill', '-s', '0', pid]) else False
except IOError:
return False


def module_exists(module_name):
try:
__import__(module_name)
except ImportError:
return False
else:
return True


@cli.command()
@click.option('--site-title',
default=config.SITE_TITLE,
prompt='Enter site title.')
@click.option('--base_url',
default=config.BASE_URL,
prompt='Enter base URL.')
@click.option('--port',
default=config.PORT,
prompt='Enter port number.')
@click.option('--secret-key',
default=config.SECRET_KEY if config.SECRET_KEY != "CHANGE_ME" else random_string(64),
prompt='Enter secret key.')
@click.option('--wiki-path',
default=config.WIKI_PATH,
prompt='Where do you want to store wiki data?',
help='Wiki Directory (git repo)')
@click.option('--allow-anon',
default=config.ALLOW_ANON,
is_flag=True,
prompt='Allow anonymous edits?')
@click.option('--registration-enabled',
default=config.REGISTRATION_ENABLED,
is_flag=True,
prompt='Enable registration?')
@click.option('--cache-type',
default=config.CACHE_TYPE,
type=click.Choice([None, 'simple', 'redis', 'memcached']),
prompt='Cache type?')
@click.option('--db-uri',
default=config.DB_URI,
prompt='Database URI? Examples: http://goo.gl/RyW0cl')
@click.pass_context
def setup(ctx, **kw):
""" Start setup wizard
"""

if not in_virtualenv() and not is_su():
# This does not account for people the have user level python installs
# that aren't virtual environments! Should be rare I think
click.secho("Setup requires root privileges, use sudo or run as root")
return

conf = {}

for k, v in kw.items():
conf[k.upper()] = v

conf_path = config.update(conf)

if conf['CACHE_TYPE'] == 'redis':
ctx.invoke(setup_redis)
elif conf['CACHE_TYPE'] == 'memcached':
ctx.invoke(setup_memcached)

click.secho('Config saved to %s' % conf_path, fg='green')
click.secho('Type "realms-wiki start" to start server', fg='yellow')
click.secho('Type "realms-wiki dev" to start server in development mode', fg='yellow')


@click.command()
@click.option('--cache-redis-host',
default=getattr(config, 'CACHE_REDIS_HOST', "127.0.0.1"),
prompt='Redis host')
@click.option('--cache-redis-port',
default=getattr(config, 'CACHE_REDIS_POST', 6379),
prompt='Redis port')
@click.option('--cache-redis-password',
default=getattr(config, 'CACHE_REDIS_PASSWORD', None),
prompt='Redis password')
@click.option('--cache-redis-db',
default=getattr(config, 'CACHE_REDIS_DB', 0),
prompt='Redis db')
def setup_redis(**kw):
conf = {}

for k, v in kw.items():
conf[k.upper()] = v

config.update(conf)
install_redis()


def get_prefix():
return sys.prefix


def get_pip():
""" Get virtualenv path for pip
"""
if not in_virtualenv() and not is_su():
click.secho("This command requires root, use sudo or run as root")
return

if in_virtualenv():
return get_prefix() + '/bin/pip'
else:
return 'pip'


@cli.command()
@click.argument('cmd', nargs=-1)
def pip(cmd):
""" Execute pip commands, useful for virtualenvs
"""
call(get_pip() + ' ' + ' '.join(cmd), shell=True)


def install_redis():
call([get_pip(), 'install', 'redis'])


def install_mysql():
call([get_pip(), 'install', 'MySQL-Python'])


def install_postgres():
call([get_pip(), 'install', 'psycopg2'])


def install_memcached():
call([get_pip(), 'install', 'python-memcached'])


@click.command()
@click.option('--cache-memcached-servers',
default=getattr(config, 'CACHE_MEMCACHED_SERVERS', ["127.0.0.1:11211"]),
type=click.STRING,
prompt='Memcached servers, separate with a space')
def setup_memcached(**kw):
conf = {}

for k, v in kw.items():
conf[k.upper()] = v

config.update(conf)


@cli.command()
@click.option('--user',
default=get_user(),
type=click.STRING,
prompt='Run as which user? (it must exist)')
@click.option('--port',
default=config.PORT,
type=click.INT,
prompt='What port to listen on?')
@click.option('--workers',
default=cpu_count() * 2 + 1,
type=click.INT,
prompt="Number of workers? (defaults to ncpu*2+1)")
def setup_upstart(**kwargs):
""" Start upstart conf creation wizard
"""
from realms.lib.util import upstart_script
import realms

print os.path.dirname(realms.__file__)

if not is_su():
click.secho("Please run this command as root or use sudo", fg='red')
return

if in_virtualenv():
app_dir = get_prefix()
path = '/'.join(sys.executable.split('/')[:-1])
else:
# Assumed root install, not sure if this matters?
app_dir = '/'
path = None

kwargs.update(dict(app_dir=app_dir, path=path))

conf_file = '/etc/init/realms-wiki.conf'

with open('/etc/init/realms-wiki.conf', 'w') as f:
f.write(upstart_script(**kwargs))

click.secho('Wrote file to %s' % conf_file, fg='yellow')
click.echo("Type 'sudo start realms-wiki' to start")
click.echo("Type 'sudo stop realms-wiki' to stop")
click.echo("Type 'sudo restart realms-wiki' to restart")


@cli.command()
@click.argument('json_string')
def configure(json_string):
""" Set config.json, expects JSON encoded string
"""
try:
config.update(json.loads(json_string))
except ValueError, e:
click.secho('Config value should be valid JSON', fg='red')


@cli.command()
@click.option('--port', default=5000)
def dev(port):
""" Run development server
"""
click.secho("Starting development server", fg='green')
app.run(host="0.0.0.0",
port=port,
debug=True)


def start_server():
if get_pid():
click.echo("Server is already running")
return

flags = '--daemon --pid %s' % config.PIDFILE

click.secho("Server started. Port: %s" % config.PORT, fg='green')

Popen('gunicorn realms:app -b 0.0.0.0:%s -k gevent %s' %
(config.PORT, flags), shell=True, executable='/bin/bash')


def stop_server():
pid = get_pid()
if not pid:
click.echo("Server is not running")
else:
click.echo("Shutting down server")
call(['kill', pid])


@cli.command()
def run():
""" Run production server (alias for start)
"""
start_server()


@cli.command()
def start():
""" Run server daemon
"""
start_server()


@cli.command()
def stop():
""" Stop server
"""
stop_server()


@cli.command()
def restart():
""" Restart server
"""
stop_server()
start_server()


@cli.command()
def status():
""" Get server status
"""
pid = get_pid()
if not pid:
click.echo("Server is not running")
else:
click.echo("Server is running PID: %s" % pid)


@cli.command()
def create_db():
""" Creates DB tables
"""
click.echo("Creating all tables")
db.create_all()


@cli.command()
@click.confirmation_option(help='Are you sure you want to drop the db?')
def drop_db():
""" Drops DB tables
"""
click.echo("Dropping all tables")
db.drop_all()


@cli.command()
def test():
""" Run tests
"""
for mod in [('flask.ext.testing', 'Flask-Testing'), ('nose', 'nose')]:
if not module_exists(mod[0]):
call([get_pip(), 'install', mod[1]])

nosetests = get_prefix() + "/bin/nosetests" if in_virtualenv() else "nosetests"

call([nosetests, config.APP_PATH])


@cli.command()
def version():
""" Output version
"""
with open('VERSION') as f:
return f.read().strip()

if __name__ == '__main__':
cli()

+ 32
- 27
realms/config/__init__.py View File

@@ -6,28 +6,48 @@ from urlparse import urlparse
def update(data):
conf = read()
conf.update(data)
save(data)
return save(data)


def read():
conf = dict()

try:
with open(os.path.join(APP_PATH, 'config.json')) as f:
conf = json.load(f)
except IOError:
pass
for k, v in os.environ.items():
if k.startswith('REALMS_'):
conf[k[7:]] = v

for loc in os.curdir, os.path.expanduser("~"), "/etc/realms-wiki", os.environ.get("REALMS_WIKI_CONF"):
try:
if not loc:
continue
with open(os.path.join(loc, "realms-wiki.json")) as f:
conf.update(json.load(f))
break
except IOError:
pass

for k in ['APP_PATH', 'USER_HOME']:
if k in conf:
del conf[k]

return conf


def save(conf):
with open(os.path.join(APP_PATH, 'config.json'), 'w') as f:
f.write(json.dumps(conf, sort_keys=True, indent=4, separators=(',', ': ')).strip() + '\n')
for loc in "/etc/realms-wiki", os.path.expanduser("~"), os.curdir:
try:
with open(os.path.join(loc, 'realms-wiki.json'), 'w') as f:
f.write(json.dumps(conf, sort_keys=True, indent=4, separators=(',', ': ')).strip() + '\n')
return os.path.join(loc, 'realms-wiki.json')
except IOError:
pass

APP_PATH = os.path.abspath(os.path.dirname(__file__) + "/../..")
USER_HOME = os.path.abspath(os.path.expanduser("~"))

# Best to change to /var/run
PIDFILE = "/tmp/realms-wiki.pid"

ENV = 'DEV'

DEBUG = True
@@ -39,7 +59,7 @@ BASE_URL = 'http://localhost'
SITE_TITLE = "Realms"

# https://pythonhosted.org/Flask-SQLAlchemy/config.html#connection-uri-format
DB_URI = 'sqlite:///%s/wiki.db' % USER_HOME
DB_URI = 'sqlite:////tmp/wiki.db'
# DB_URI = 'mysql://scott:tiger@localhost/mydatabase'
# DB_URI = 'postgresql://scott:tiger@localhost/mydatabase'
# DB_URI = 'oracle://scott:tiger@127.0.0.1:1521/sidname'
@@ -67,7 +87,7 @@ RECAPTCHA_OPTIONS = {}
SECRET_KEY = 'CHANGE_ME'

# Path on file system where wiki data will reside
WIKI_PATH = os.path.join(APP_PATH, 'wiki')
WIKI_PATH = '/tmp/wiki'

# Name of page that will act as home
WIKI_HOME = 'home'
@@ -78,7 +98,7 @@ REGISTRATION_ENABLED = True
# Used by Flask-Login
LOGIN_DISABLED = ALLOW_ANON

# None, firepad, or togetherjs
# None, firepad, and/or togetherjs
COLLABORATION = 'togetherjs'

# Required for firepad
@@ -91,22 +111,7 @@ LOCKED = WIKI_LOCKED_PAGES

ROOT_ENDPOINT = 'wiki.page'

__env = {}
for k, v in os.environ.items():
if k.startswith('REALMS_'):
__env[k[7:]] = v

globals().update(__env)

try:
with open(os.path.join(APP_PATH, 'config.json')) as f:
__settings = json.load(f)
for k in ['APP_PATH', 'USER_HOME']:
if k in __settings:
del __settings[k]
globals().update(__settings)
except IOError:
pass
globals().update(read())

if BASE_URL.endswith('/'):
BASE_URL = BASE_URL[-1]


+ 47
- 0
realms/lib/util.py View File

@@ -4,6 +4,7 @@ import hashlib
import json
import string
import random
from jinja2 import Template


class AttrDict(dict):
@@ -97,3 +98,49 @@ def to_canonical(s):

def gravatar_url(email):
return "//www.gravatar.com/avatar/" + hashlib.md5(email).hexdigest()

def upstart_script(user='root', app_dir=None, port=5000, workers=2, path=None):
script = """
limit nofile 65335 65335

respawn

description "Realms Wiki"
author "scragg@gmail.com"

chdir {{ app_dir }}

{% if path %}
env PATH={{ path }}:/usr/local/bin:/usr/bin:/bin:$PATH
export PATH
{% endif %}

env LC_ALL=en_US.UTF-8
env GEVENT_RESOLVER=ares

export LC_ALL
export GEVENT_RESOLVER

setuid {{ user }}
setgid {{ user }}

start on runlevel [2345]
stop on runlevel [!2345]

respawn

exec gunicorn \
--name realms-wiki \
--access-logfile - \
--error-logfile - \
--worker-class gevent \
--workers {{ workers }} \
--bind 0.0.0.0:{{ port }} \
--user {{ user }} \
--group {{ user }} \
--chdir {{ app_dir }} \
realms:app

"""
template = Template(script)
return template.render(user=user, app_dir=app_dir, port=port, workers=workers, path=path)

+ 2
- 0
realms/modules/wiki/tests.py View File

@@ -15,9 +15,11 @@ class WikiTest(TestCase):

self.assert_200(self.client.get(url_for("wiki.create")))

""" Create a test page first!
for route in ['page', 'edit', 'history', 'index']:
rv = self.client.get(url_for("wiki.%s" % route, name='test'))
self.assert_200(rv, "wiki.%s: %s" % (route, rv.status_code))
"""

def test_write_page(self):
pass


dev-requirements.txt → requirements-dev.txt View File


+ 1
- 0
requirements.txt View File

@@ -11,6 +11,7 @@ click==3.3
gevent==1.0.1
ghdiff==0.4
gittle==0.4.0
gunicorn==19.1.1
itsdangerous==0.24
lxml==3.4.0
markdown2==2.3.0


+ 13
- 4
setup.py View File

@@ -1,8 +1,12 @@
from setuptools import setup, find_packages
import os

if os.environ.get('USER', '') == 'vagrant':
del os.link

DESCRIPTION = "Simple git based wiki"

with open('README.md') as f:
with open('README') as f:
LONG_DESCRIPTION = f.read()

with open('requirements.txt') as f:
@@ -13,15 +17,20 @@ with open('VERSION') as f:

CLASSIFIERS = [
'Intended Audience :: Developers',
'License :: OSI Approved :: GPLv2 License',
'Operating System :: OS Independent',
'License :: OSI Approved :: GNU General Public License v2 (GPLv2)',
'Operating System :: POSIX :: Linux',
'Programming Language :: Python',
'Topic :: Software Development :: Libraries :: Python Modules']
'Topic :: Internet :: WWW/HTTP :: Dynamic Content']

setup(name='realms-wiki',
version=VERSION,
packages=find_packages(),
install_requires=required,
#scripts=['realms-wiki'],
entry_points={
'console_scripts': [
'realms-wiki = realms.cli:cli'
]},
author='Matthew Scragg',
author_email='scragg@gmail.com',
maintainer='Matthew Scragg',


Loading…
Cancel
Save