init

pahaz · Jul 23, 2016 · 3efa614 · 3efa614
commit 3efa614
Show file tree

Hide file tree

Showing 12 changed files with 409 additions and 0 deletions.
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Pahaz Blinov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -0,0 +1,8 @@
+# Include the data files recursive-include data *
+
+# If using Python 2.6 or less, then have to include package data, even though
+# it's already declared in setup.py
+# include sample/*.dat
+
+include LICENSE
+include *.rst
diff --git a/README.rst b/README.rst
@@ -0,0 +1,76 @@
+**Author**: `Pahaz Blinov`_
+
+**Repo**: https://github.com/pahaz/py3line/
+
+Pyline is a UNIX command-line tool for line-based processing
+in Python with regex and output transform features
+similar to grep, sed, and awk.
+
+This project inspired by: `piep`_, `pysed`_, `pyline`_, `pyp`_ and
+`Jacob+Mark recipe <https://code.activestate.com/recipes/437932-pyline-a-grep-like-sed-like-command-line-tool/>`_
+
+**requirements**: Python3
+
+**WHY I MAKE IT?**
+
+I sometimes have to use `sed` / `awk`.
+Not often, and so I always forget the necessary options and `sed` / `awk` DSL.
+But I now python, I like it, and I want use it for data processing.
+Default `python -c` is hard to write the kind of one-liner that works well.
+
+Why not a `pyline`?
+ * Don`t support python3
+ * Have many options (I want as much simple as possible solution)
+ * Bad performance
+ * Don`t support command chaining
+
+Why not a `pysed`?
+ *
+
+Installation
+============
+
+`py3line`_ is on PyPI, so simply run:
+
+::
+
+    pip install py3line
+
+or ::
+
+    easy_install py3line
+
+to have it installed in your environment.
+
+For installing from source, clone the
+`repo <https://github.com/pahaz/py3line>`_ and run::
+
+    python setup.py install
+
+Usage scenarios
+===============
+
+...
+
+Examples
+--------
+
+Example 1: create spreadsheet
+=============================
+
+.. code-block:: bash
+
+    $ echo -e "Here are\nsome\nwords for you." | ./py3line.py "x.split()" -a "len(x)"
+    2
+    1
+    3
+
+DOCS
+----
+
+.. _Pahaz Blinov: https://github.com/pahaz/
+.. _py3line: https://pypi.python.org/pypi/py3line/
+.. _pyp: https://pypi.python.org/pypi/pyp/
+.. _piep: https://github.com/timbertson/piep/tree/master/piep/
+.. _pysed: https://github.com/dslackw/pysed/blob/master/pysed/main.py
+.. _pyline: https://github.com/westurner/pyline/blob/master/pyline/pyline.py
diff --git a/bashtest.py b/bashtest.py
@@ -0,0 +1,118 @@
+import argparse
+import doctest
+import re
+import shlex
+import sys
+import types
+
+
+__version__ = '0.0.1'
+NAME = 'bashtest'
+CHECK_EXITCODE = False
+
+
+def _fake_module_run(command):
+    import subprocess
+    import shlex
+    command = 'bash -c %s' % (shlex.quote(command))
+    r = subprocess.run(command, stdout=subprocess.PIPE,
+                       stderr=subprocess.STDOUT, shell=True)
+    print(r.stdout.decode())
+    if CHECK_EXITCODE:
+        print('<EXITCODE={}>'.format(r.returncode))
+
+
+def parseargs():
+    description = (
+        "BashTest is a UNIX command-line tool for running text-based bash "
+        "tests. "
+    )
+
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument('files', metavar='file', nargs='+',
+                        help='Input file')
+    parser.add_argument('--exitcode',
+                        action='store_true',
+                        help='Print exitcode after command end of output')
+    parser.add_argument('--no-blankline-substitution',  # I do not use it
+                        dest='no_blankline_substitution',
+                        action='store_true',
+                        help='Substitute `<BLANKLINE>` if an expected output '
+                             'block contains a line containing only the `\\n`')
+    parser.add_argument('--no-normalize-whitespace',
+                        dest='no_normalize_whitespace',
+                        action='store_true',
+                        help='All sequences of whitespace '
+                             '(blanks and newlines) are not equal')
+
+    parser.add_argument('-v', '--verbose',
+                        dest='verbose',
+                        action='store_true')
+    parser.add_argument('-q', '--quiet',
+                        dest='quiet',
+                        action='store_true')
+
+    parser.add_argument('--version',
+                        dest='version',
+                        action='store_true',
+                        help='Print the version string')
+
+    return parser.parse_args()
+
+
+def __re_repl(match):
+    g1 = match.group(1)
+    g2 = match.group(2)
+    return '%s>>> run(%s)' % (g1, shlex.quote(g2.replace('\\', '\\\\')))
+
+
+def main():
+    global CHECK_EXITCODE
+
+    args = parseargs()
+    optionflags = 0
+
+    if args.version:
+        print(__version__)
+        return 0
+
+    if args.quiet:
+        args.verbose = False
+
+    if args.exitcode:
+        CHECK_EXITCODE = True
+
+    if args.no_blankline_substitution:
+        optionflags |= doctest.DONT_ACCEPT_BLANKLINE
+
+    if not args.no_normalize_whitespace:
+        optionflags |= doctest.NORMALIZE_WHITESPACE
+
+    finder = doctest.DocTestFinder()
+    runner = doctest.DocTestRunner(
+        verbose=args.verbose, optionflags=optionflags)
+
+    ret = 0
+    rgx = re.compile('^(\s*)\$ (.+)$')
+    for file in args.files:
+        with open(file) as f:
+            res = []
+            for line in f:
+                if rgx.match(line):
+                    line, _ = rgx.subn(__re_repl, line)
+                res.append(line)
+
+        res = ''.join(res)
+        fake_module = types.ModuleType(file, res)
+        fake_module.run = _fake_module_run
+        for test in finder.find(fake_module, file):
+            runner.run(test)
+
+    runner.summarize(verbose=not args.quiet)
+
+    if runner.failures:
+        ret = 1
+    return ret
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/setup.py b/setup.py
@@ -0,0 +1,124 @@
+"""A setuptools based setup module.
+
+See:
+https://packaging.python.org/en/latest/distributing.html
+https://github.com/pypa/sampleproject
+"""
+
+import re
+from os import path
+from codecs import open  # To use a consistent encoding
+
+from setuptools import setup  # Always prefer setuptools over distutils
+
+here = path.abspath(path.dirname(__file__))
+name = 'bashtest'
+description = 'UNIX command-line tool for python line-based stream processing'
+url = 'https://github.com/pahaz/bashtest'
+ppa = 'https://pypi.python.org/packages/source/s/{0}/{0}-'.format(name)
+
+# Get the long description from the README file
+with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
+    long_description = f.read()
+
+with open(path.join(here, name + '.py'), encoding='utf-8') as f:
+    data = f.read()
+    version = eval(re.search("__version__[ ]*=[ ]*([^\r\n]+)", data).group(1))
+
+setup(
+    name=name,
+
+    # Versions should comply with PEP440.  For a discussion on single-sourcing
+    # the version across setup.py and the project code, see
+    # https://packaging.python.org/en/latest/single_source_version.html
+    version=version,
+
+    description=description,
+    long_description=long_description,
+
+    # The project's main homepage.
+    url=url,
+    download_url=ppa + version + '.zip',  # noqa
+
+    # Author details
+    author='Pahaz Blinov',
+    author_email='[email protected]',
+
+    # Choose your license
+    license='MIT',
+
+    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
+    classifiers=[
+        # How mature is this project? Common values are
+        #   3 - Alpha
+        #   4 - Beta
+        #   5 - Production/Stable
+        'Development Status :: 3 - Alpha',
+
+        # Indicate who your project is intended for
+        'Intended Audience :: Developers',
+        'Topic :: Utilities',
+
+        # Pick your license as you wish (should match "license" above)
+        'License :: OSI Approved :: MIT License',
+
+        # Specify the Python versions you support here. In particular, ensure
+        # that you indicate whether you support Python 2, Python 3 or both.
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+    ],
+
+    platforms=['unix', 'macos', 'windows'],
+
+    # What does your project relate to?
+    keywords='google spreadsheet api util helper',
+
+    # You can just specify the packages manually here if your project is
+    # simple. Or you can use find_packages().
+    # packages=find_packages(exclude=['contrib', 'docs', 'tests']),
+
+    # Alternatively, if you want to distribute just a my_module.py, uncomment
+    # this:
+    py_modules=["%s" % name],
+
+    # List run-time dependencies here.  These will be installed by pip when
+    # your project is installed. For an analysis of "install_requires" vs pip's
+    # requirements files see:
+    # https://packaging.python.org/en/latest/requirements.html
+    install_requires=[
+    ],
+
+    # List additional groups of dependencies here (e.g. development
+    # dependencies). You can install these using the following syntax,
+    # for example:
+    # $ pip install -e .[dev,test]
+    extras_require={
+        'dev': ['check-manifest', 'docutils', 'Pygments'],
+        # 'test': [
+        #     'tox>=1.8.1',
+        # ],
+        # 'build_sphinx': [
+        #     'sphinx',
+        #     'sphinxcontrib-napoleon',
+        # ],
+    },
+
+    # If there are data files included in your packages that need to be
+    # installed, specify them here.  If using Python 2.6 or less, then these
+    # have to be included in MANIFEST.in as well.
+    # package_data={
+    #     'tests': ['testrsa.key'],
+    # },
+
+    # To provide executable scripts, use entry points in preference to the
+    # "scripts" keyword. Entry points provide cross-platform support and allow
+    # pip to create the appropriate form of executable for the target platform.
+    # entry_points={
+    #     'console_scripts': [
+    #         'sshtunnel=sshtunnel:_cli_main',
+    #     ]
+    # },
+
+    # Integrate tox with setuptools
+    # cmdclass={'test': Tox},
+)
diff --git a/testsuit/list-directory/file1 b/testsuit/list-directory/file1
@@ -0,0 +1,2 @@
+line1
+line2
diff --git a/testsuit/list-directory/file2.txt b/testsuit/list-directory/file2.txt
@@ -0,0 +1,6 @@
+This utility was born from the fact that I keep forgetting how to use "sed",
+and I suck at Perl. It brings ad-hoc command-line piping sensibilities
+to the Python interpeter.
+
+(Version 1.2 does better outputting of list-like results,
+thanks to Mark Eichin.)
diff --git a/testsuit/list-directory/file3.py b/testsuit/list-directory/file3.py
@@ -0,0 +1,2 @@
+import sys
+sys.exit(22)
diff --git a/testsuit/list-directory/file4.sh b/testsuit/list-directory/file4.sh
@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+exit 33
diff --git a/testsuit/nginx.log b/testsuit/nginx.log
@@ -0,0 +1,18 @@
+[19/Jul/2016:21:50:12 +0300] 184.73.237.85 [1/18089] uget=- uset=uuid=0C00A8C064768E57F54178C002700503 - rt="0.005" uht="0.005" urt="0.005" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOdmTAeEH1AwVwAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/"
+[19/Jul/2016:21:50:21 +0300] 50.112.95.211 [1/18091] uget=- uset=uuid=0C00A8C06D768E57F54178C002710503 - rt="0.005" uht="0.005" urt="0.005" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOdm3AeEH1AwVxAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/"
+[19/Jul/2016:21:52:09 +0300] 54.251.34.67 [1/18093] uget=- uset=uuid=0C00A8C0D9768E57F54178C002720503 - rt="0.007" uht="0.007" urt="0.007" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOdtnAeEH1AwVyAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/"
+[19/Jul/2016:21:52:35 +0300] 54.248.250.232 [1/18095] uget=- uset=uuid=0C00A8C0F3768E57F54178C002730503 - rt="0.006" uht="0.006" urt="0.006" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOdvPAeEH1AwVzAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/"
+[19/Jul/2016:21:30:21 +0300] 50.112.95.211 [1/17958] uget=- uset=uuid=0C00A8C0BD718E57F3414CC0029F0503 - rt="0.004" uht="0.004" urt="0.004" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOcb3ATEHzAwWfAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/"
+[19/Jul/2016:21:30:22 +0300] 92.248.130.149 [5/17954] uget=uuid=0C00A8C0BF940F57526A38BB02200303 uset=- - rt="0.424" uht="0.396" urt="0.410" - status=200 - "GET /admin/moktoring/session/add/ HTTP/1.1" len=672 sent=42330/41540 - "https://o7x.examus.net/admin/moktoring/session/" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" set_cookie="csrftoken=06uZBa0KnNSSsTg0q0G4ifvzZwQSBJ7n; expires=Tue, 18-Jul-2017 18:30:22 GMT; Max-Age=31449600; Path=/"
+[19/Jul/2016:21:30:23 +0300] 92.248.130.149 [6/17954] uget=uuid=0C00A8C0BF940F57526A38BB02200303 uset=- - rt="0.029" uht="0.026" urt="0.029" - status=200 - "GET /admin/jsi18n/ HTTP/1.1" len=559 sent=15167/14680 - "https://o7x.examus.net/admin/moktoring/session/add/" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" set_cookie="-"
+[19/Jul/2016:21:30:23 +0300] 92.248.130.149 [3/17950] uget=uuid=0C00A8C0BF940F57526A38BB02200303 uset=- - rt="0.000" uht="-" urt="-" - status=200 - "GET /static/admin/img/icon-calendar.svg HTTP/1.1" len=604 sent=1612/1086 - "https://o7x.examus.net/static/admin/css/widgets.css" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" set_cookie="-"
+[19/Jul/2016:21:30:23 +0300] 92.248.130.149 [7/17954] uget=uuid=0C00A8C0BF940F57526A38BB02200303 uset=- - rt="0.000" uht="-" urt="-" - status=200 - "GET /static/admin/img/icon-clock.svg HTTP/1.1" len=601 sent=1202/677 - "https://o7x.examus.net/static/admin/css/widgets.css" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" set_cookie="-"
+[19/Jul/2016:21:32:00 +0300] 54.251.34.67 [1/17963] uget=- uset=uuid=0C00A8C020728E57F3414CC002A00503 - rt="0.008" uht="0.008" urt="0.008" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOciDATEHzAwWgAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/"
+[19/Jul/2016:21:32:35 +0300] 54.248.250.232 [1/17965] uget=- uset=uuid=0C00A8C043728E57F3414CC002A10503 - rt="0.007" uht="0.007" urt="0.007" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOckPATEHzAwWhAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/"
+[19/Jul/2016:21:32:38 +0300] 184.73.237.85 [1/17967] uget=- uset=uuid=0C00A8C046728E57F3414CC002A20503 - rt="0.005" uht="0.005" urt="0.005" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOckbATEHzAwWiAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/"
+[19/Jul/2016:21:32:40 +0300] 54.247.188.179 [1/17969] uget=- uset=uuid=0C00A8C048728E57F3414CC002A30503 - rt="0.007" uht="0.007" urt="0.007" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOckjATEHzAwWjAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/"
+[19/Jul/2016:21:32:51 +0300] 50.112.95.211 [1/17971] uget=- uset=uuid=0C00A8C053728E57F3414CC002A40503 - rt="0.005" uht="0.005" urt="0.005" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOclPATEHzAwWkAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/"
+[19/Jul/2016:21:33:42 +0300] 109.173.101.89 [1/17973] uget=- uset=uuid=0C00A8C086728E57F3414CC002A50503 - rt="0.009" uht="0.005" urt="0.009" - status=302 - "GET /logout/?reason=startApplication HTTP/1.1" len=433 sent=606/5 - "-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" set_cookie="uuid=wKgADFeOcobATEHzAwWlAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/"
+[19/Jul/2016:21:33:42 +0300] 109.173.101.89 [2/17973] uget=uuid=0C00A8C086728E57F3414CC002A50503 uset=- - rt="0.010" uht="0.006" urt="0.010" - status=302 - "GET / HTTP/1.1" len=442 sent=530/5 - "-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" set_cookie="-"
+[19/Jul/2016:21:33:42 +0300] 109.173.101.89 [3/17973] uget=uuid=0C00A8C086728E57F3414CC002A50503 uset=- - rt="0.047" uht="0.039" urt="0.047" - status=200 - "GET /login/?next=/ HTTP/1.1" len=455 sent=8370/7580 - "-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" set_cookie="csrftoken=xZxfdkNjadcYIQXcKg8Wv0x8nQq9NwL0; expires=Tue, 18-Jul-2017 18:33:42 GMT; Max-Age=31449600; Path=/"
+[19/Jul/2016:20:09:50 +0300] 92.248.130.149 [22/17536] uget=uuid=0C00A8C0BF940F57526A38BB02200303 uset=- - rt="0.078" uht="0.072" urt="0.078" - status=302 - "POST /admin/customauth/user/?q=%D0%9F%D0%B0%D1%81%D0%B5%D1%87%D0%BD%D0%B8%D0%BA HTTP/1.1" len=1189 sent=1063/5 - "https://o7x.examus.net/admin/customauth/user/?q=%D0%9F%D0%B0%D1%81%D0%B5%D1%87%D0%BD%D0%B8%D0%BA" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" set_cookie="messages=\x2292939612ff14300f94f8722c99618e22187e3503$[[\x5C\x22__json_message\x5C\x22\x5C0540\x5C05425\x5C054\x5C\x221 \x5C\x5Cu041f\x5C\x5Cu043e\x5C\x5Cu043b\x5C\x5Cu044c\x5C\x5Cu0437\x5C\x5Cu043e\x5C\x5Cu0432\x5C\x5Cu0430\x5C\x5Cu0442\x5C\x5Cu0435\x5C\x5Cu043b\x5C\x5Cu044c \x5C\x5Cu0431\x5C\x5Cu044b\x5C\x5Cu043b \x5C\x5Cu0443\x5C\x5Cu0441\x5C\x5Cu043f\x5C\x5Cu0435\x5C\x5Cu0448\x5C\x5Cu043d\x5C\x5Cu043e \x5C\x5Cu0438\x5C\x5Cu0437\x5C\x5Cu043c\x5C\x5Cu0435\x5C\x5Cu043d\x5C\x5Cu0435\x5C\x5Cu043d.\x5C\x22]]\x22; Path=/"
diff --git a/testsuit/test.text b/testsuit/test.text
@@ -0,0 +1,8 @@
+This is my cat,
+ whose name is Betty.
+This is my dog,
+ whose name is Frank.
+This is my fish,
+ whose name is George.
+This is my goat,
+ whose name is Adam.