First commit

vendor/psy/psysh/test/tools/gen_unvis_fixtures.py

@@ -0,0 +1,94 @@
+#! /usr/bin/env python3
+import sys
+from os.path import abspath, expanduser, dirname, join
+from itertools import chain
+import json
+import argparse
+
+from vis import vis, unvis, VIS_WHITE
+
+
+__dir__ = dirname(abspath(__file__))
+
+OUTPUT_FILE = join(__dir__, '..', 'fixtures', 'unvis_fixtures.json')
+
+# Add custom fixtures here
+CUSTOM_FIXTURES = [
+    # test long multibyte string
+    ''.join(chr(cp) for cp in range(1024)),
+    'foo bar',
+    'foo\nbar',
+    "$bar = 'baz';",
+    r'$foo = "\x20\\x20\\\x20\\\\x20"',
+    '$foo = function($bar) use($baz) {\n\treturn $baz->getFoo()\n};'
+]
+
+RANGES = {
+    # All valid codepoints in the BMP
+    'bmp': chain(range(0x0000, 0xD800), range(0xE000, 0xFFFF)),
+    # Smaller set of pertinent? codepoints inside BMP
+    # see: http://en.wikipedia.org/wiki/Plane_(Unicode)#Basic_Multilingual_Plane
+    'small': chain(
+        # latin blocks
+        range(0x0000, 0x0250),
+        # Greek, Cyrillic
+        range(0x0370, 0x0530),
+        # Hebrew, Arabic
+        range(0x590, 0x0700),
+        # CJK radicals
+        range(0x2E80, 0x2F00),
+        # Hiragana, Katakana
+        range(0x3040, 0x3100)
+    )
+}
+
+
+if __name__ == '__main__':
+
+    argp = argparse.ArgumentParser(
+        description='Generates test data for Psy\\Test\\Util\\StrTest')
+    argp.add_argument('-f', '--format-output', action='store_true',
+                      help='Indent JSON output to ease debugging')
+    argp.add_argument('-a', '--all', action='store_true',
+                      help="""Generates test data for all codepoints of the BMP.
+                      (same as --range=bmp). WARNING: You will need quite
+                      a lot of RAM to run the testsuite !
+                      """)
+    argp.add_argument('-r', '--range',
+                      help="""Choose the range of codepoints used to generate
+                      test data.""",
+                      choices=list(RANGES.keys()),
+                      default='small')
+    argp.add_argument('-o', '--output-file',
+                      help="""Write test data to OUTPUT_FILE
+                      (defaults to PSYSH_DIR/test/fixtures)""")
+    args = argp.parse_args()
+
+    cp_range = RANGES['bmp'] if args.all else RANGES[args.range]
+    indent = 2 if args.format_output else None
+    if args.output_file:
+        OUTPUT_FILE = abspath(expanduser(args.output_file))
+
+    fixtures = []
+
+    # use SMALL_RANGE by default, it should be enough.
+    # use BMP_RANGE for a more complete smoke test
+    for codepoint in cp_range:
+        char = chr(codepoint)
+        encoded = vis(char, VIS_WHITE)
+        decoded = unvis(encoded)
+        fixtures.append((encoded, decoded))
+
+    # Add our own custom fixtures at the end,
+    # since they would fail anyway if one of the previous did.
+    for fixture in CUSTOM_FIXTURES:
+        encoded = vis(fixture, VIS_WHITE)
+        decoded = unvis(encoded)
+        fixtures.append((encoded, decoded))
+
+    with open(OUTPUT_FILE, 'w') as fp:
+        # dump as json to avoid backslashin and quotin nightmare
+        # between php and python
+        json.dump(fixtures, fp, indent=indent)
+
+    sys.exit(0)

vendor/psy/psysh/test/tools/vis.py

@@ -0,0 +1,126 @@
+"""
+vis.py
+======
+
+Ctypes based module to access libbsd's strvis & strunvis functions.
+
+The `vis` function is the equivalent of strvis.
+The `unvis` function is the equivalent of strunvis.
+All functions accept unicode string as input and return a unicode string.
+
+Constants:
+----------
+
+* to select alternate encoding format
+  `VIS_OCTAL`:      use octal \ddd format
+  `VIS_CSTYLE`:     use \[nrft0..] where appropiate
+
+* to alter set of characters encoded
+  (default is to encode all non-graphic except space, tab, and newline).
+  `VIS_SP`:         also encode space
+  `VIS_TAB`:        also encode tab
+  `VIS_NL`:         also encode newline
+  `VIS_WHITE`:      same as (VIS_SP | VIS_TAB | VIS_NL)
+  `VIS_SAFE`:       only encode "unsafe" characters
+
+* other
+  `VIS_NOSLASH`:    inhibit printing '\'
+  `VIS_HTTP1808`:   http-style escape % hex hex
+  `VIS_HTTPSTYLE`:  http-style escape % hex hex
+  `VIS_MIMESTYLE`:  mime-style escape = HEX HEX
+  `VIS_HTTP1866`:   http-style &#num; or &string;
+  `VIS_NOESCAPE`:   don't decode `\'
+  `VIS_GLOB`:       encode glob(3) magic characters
+
+:Authors:
+    - ju1ius (http://github.com/ju1ius)
+:Version: 1
+:Date: 2014-01-05
+"""
+from ctypes import CDLL, c_char_p, c_int
+from ctypes.util import find_library
+
+
+__all__ = [
+    'vis', 'unvis',
+    'VIS_OCTAL', 'VIS_CSTYLE',
+    'VIS_SP', 'VIS_TAB', 'VIS_NL', 'VIS_WHITE', 'VIS_SAFE',
+    'VIS_NOSLASH', 'VIS_HTTP1808', 'VIS_HTTPSTYLE', 'VIS_MIMESTYLE',
+    'VIS_HTTP1866', 'VIS_NOESCAPE', 'VIS_GLOB'
+]
+
+
+#############################################################
+# Constants from bsd/vis.h
+#############################################################
+
+#to select alternate encoding format
+VIS_OCTAL = 0x0001
+VIS_CSTYLE = 0x0002
+# to alter set of characters encoded
+# (default is to encode all non-graphic except space, tab, and newline).
+VIS_SP = 0x0004
+VIS_TAB = 0x0008
+VIS_NL = 0x0010
+VIS_WHITE = VIS_SP | VIS_TAB | VIS_NL
+VIS_SAFE = 0x0020
+# other
+VIS_NOSLASH = 0x0040
+VIS_HTTP1808 = 0x0080
+VIS_HTTPSTYLE = 0x0080
+VIS_MIMESTYLE = 0x0100
+VIS_HTTP1866 = 0x0200
+VIS_NOESCAPE = 0x0400
+VIS_GLOB = 0x1000
+
+#############################################################
+# Import libbsd/vis functions
+#############################################################
+
+_libbsd = CDLL(find_library('bsd'))
+
+_strvis = _libbsd.strvis
+_strvis.argtypes = [c_char_p, c_char_p, c_int]
+_strvis.restype = c_int
+
+_strunvis = _libbsd.strunvis
+_strvis.argtypes = [c_char_p, c_char_p]
+_strvis.restype = c_int
+
+
+def vis(src, flags=VIS_WHITE):
+    """
+    Encodes the string `src` into libbsd's vis encoding.
+    `flags` must be one of the VIS_* constants
+
+    C definition:
+    int strvis(char *dst, char *src, int flags);
+    """
+    src = bytes(src, 'utf-8')
+    dst_p = c_char_p(bytes(len(src) * 4))
+    src_p = c_char_p(src)
+    flags = c_int(flags)
+
+    bytes_written = _strvis(dst_p, src_p, flags)
+    if -1 == bytes_written:
+        raise RuntimeError('vis failed to encode string "{}"'.format(src))
+
+    return dst_p.value.decode('utf-8')
+
+
+def unvis(src):
+    """
+    Decodes a string encoded by vis.
+
+    C definition:
+    int strunvis(char *dst, char *src);
+    """
+    src = bytes(src, 'utf-8')
+    dst_p = c_char_p(bytes(len(src)))
+    src_p = c_char_p(src)
+
+    bytes_written = _strunvis(dst_p, src_p)
+    if -1 == bytes_written:
+        raise RuntimeError('unvis failed to decode string "{}"'.format(src))
+
+    return dst_p.value.decode('utf-8')