diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ecfc47b --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Pahaz Blinov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..6e336f0 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,8 @@ +# Include the data files recursive-include data * + +# If using Python 2.6 or less, then have to include package data, even though +# it's already declared in setup.py +# include sample/*.dat + +include LICENSE +include *.rst diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..eed66f6 --- /dev/null +++ b/README.rst @@ -0,0 +1,76 @@ +**Author**: `Pahaz Blinov`_ + +**Repo**: https://github.com/pahaz/py3line/ + +Pyline is a UNIX command-line tool for line-based processing +in Python with regex and output transform features +similar to grep, sed, and awk. + +This project inspired by: `piep`_, `pysed`_, `pyline`_, `pyp`_ and +`Jacob+Mark recipe `_ + +**requirements**: Python3 + +**WHY I MAKE IT?** + +I sometimes have to use `sed` / `awk`. +Not often, and so I always forget the necessary options and `sed` / `awk` DSL. +But I now python, I like it, and I want use it for data processing. +Default `python -c` is hard to write the kind of one-liner that works well. + +Why not a `pyline`? + * Don`t support python3 + * Have many options (I want as much simple as possible solution) + * Bad performance + * Don`t support command chaining + +Why not a `pysed`? + * + +Installation +============ + +`py3line`_ is on PyPI, so simply run: + +:: + + pip install py3line + +or :: + + easy_install py3line + +to have it installed in your environment. + +For installing from source, clone the +`repo `_ and run:: + + python setup.py install + +Usage scenarios +=============== + +... + +Examples +-------- + +Example 1: create spreadsheet +============================= + +.. code-block:: bash + + $ echo -e "Here are\nsome\nwords for you." | ./py3line.py "x.split()" -a "len(x)" + 2 + 1 + 3 + +DOCS +---- + +.. _Pahaz Blinov: https://github.com/pahaz/ +.. _py3line: https://pypi.python.org/pypi/py3line/ +.. _pyp: https://pypi.python.org/pypi/pyp/ +.. _piep: https://github.com/timbertson/piep/tree/master/piep/ +.. _pysed: https://github.com/dslackw/pysed/blob/master/pysed/main.py +.. _pyline: https://github.com/westurner/pyline/blob/master/pyline/pyline.py diff --git a/bashtest.py b/bashtest.py new file mode 100644 index 0000000..a7c4442 --- /dev/null +++ b/bashtest.py @@ -0,0 +1,118 @@ +import argparse +import doctest +import re +import shlex +import sys +import types + + +__version__ = '0.0.1' +NAME = 'bashtest' +CHECK_EXITCODE = False + + +def _fake_module_run(command): + import subprocess + import shlex + command = 'bash -c %s' % (shlex.quote(command)) + r = subprocess.run(command, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, shell=True) + print(r.stdout.decode()) + if CHECK_EXITCODE: + print(''.format(r.returncode)) + + +def parseargs(): + description = ( + "BashTest is a UNIX command-line tool for running text-based bash " + "tests. " + ) + + parser = argparse.ArgumentParser(description=description) + parser.add_argument('files', metavar='file', nargs='+', + help='Input file') + parser.add_argument('--exitcode', + action='store_true', + help='Print exitcode after command end of output') + parser.add_argument('--no-blankline-substitution', # I do not use it + dest='no_blankline_substitution', + action='store_true', + help='Substitute `` if an expected output ' + 'block contains a line containing only the `\\n`') + parser.add_argument('--no-normalize-whitespace', + dest='no_normalize_whitespace', + action='store_true', + help='All sequences of whitespace ' + '(blanks and newlines) are not equal') + + parser.add_argument('-v', '--verbose', + dest='verbose', + action='store_true') + parser.add_argument('-q', '--quiet', + dest='quiet', + action='store_true') + + parser.add_argument('--version', + dest='version', + action='store_true', + help='Print the version string') + + return parser.parse_args() + + +def __re_repl(match): + g1 = match.group(1) + g2 = match.group(2) + return '%s>>> run(%s)' % (g1, shlex.quote(g2.replace('\\', '\\\\'))) + + +def main(): + global CHECK_EXITCODE + + args = parseargs() + optionflags = 0 + + if args.version: + print(__version__) + return 0 + + if args.quiet: + args.verbose = False + + if args.exitcode: + CHECK_EXITCODE = True + + if args.no_blankline_substitution: + optionflags |= doctest.DONT_ACCEPT_BLANKLINE + + if not args.no_normalize_whitespace: + optionflags |= doctest.NORMALIZE_WHITESPACE + + finder = doctest.DocTestFinder() + runner = doctest.DocTestRunner( + verbose=args.verbose, optionflags=optionflags) + + ret = 0 + rgx = re.compile('^(\s*)\$ (.+)$') + for file in args.files: + with open(file) as f: + res = [] + for line in f: + if rgx.match(line): + line, _ = rgx.subn(__re_repl, line) + res.append(line) + + res = ''.join(res) + fake_module = types.ModuleType(file, res) + fake_module.run = _fake_module_run + for test in finder.find(fake_module, file): + runner.run(test) + + runner.summarize(verbose=not args.quiet) + + if runner.failures: + ret = 1 + return ret + +if __name__ == '__main__': + sys.exit(main()) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d428b53 --- /dev/null +++ b/setup.py @@ -0,0 +1,124 @@ +"""A setuptools based setup module. + +See: +https://packaging.python.org/en/latest/distributing.html +https://github.com/pypa/sampleproject +""" + +import re +from os import path +from codecs import open # To use a consistent encoding + +from setuptools import setup # Always prefer setuptools over distutils + +here = path.abspath(path.dirname(__file__)) +name = 'bashtest' +description = 'UNIX command-line tool for python line-based stream processing' +url = 'https://github.com/pahaz/bashtest' +ppa = 'https://pypi.python.org/packages/source/s/{0}/{0}-'.format(name) + +# Get the long description from the README file +with open(path.join(here, 'README.rst'), encoding='utf-8') as f: + long_description = f.read() + +with open(path.join(here, name + '.py'), encoding='utf-8') as f: + data = f.read() + version = eval(re.search("__version__[ ]*=[ ]*([^\r\n]+)", data).group(1)) + +setup( + name=name, + + # Versions should comply with PEP440. For a discussion on single-sourcing + # the version across setup.py and the project code, see + # https://packaging.python.org/en/latest/single_source_version.html + version=version, + + description=description, + long_description=long_description, + + # The project's main homepage. + url=url, + download_url=ppa + version + '.zip', # noqa + + # Author details + author='Pahaz Blinov', + author_email='pahaz.blinov@gmail.com', + + # Choose your license + license='MIT', + + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + 'Development Status :: 3 - Alpha', + + # Indicate who your project is intended for + 'Intended Audience :: Developers', + 'Topic :: Utilities', + + # Pick your license as you wish (should match "license" above) + 'License :: OSI Approved :: MIT License', + + # Specify the Python versions you support here. In particular, ensure + # that you indicate whether you support Python 2, Python 3 or both. + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + ], + + platforms=['unix', 'macos', 'windows'], + + # What does your project relate to? + keywords='google spreadsheet api util helper', + + # You can just specify the packages manually here if your project is + # simple. Or you can use find_packages(). + # packages=find_packages(exclude=['contrib', 'docs', 'tests']), + + # Alternatively, if you want to distribute just a my_module.py, uncomment + # this: + py_modules=["%s" % name], + + # List run-time dependencies here. These will be installed by pip when + # your project is installed. For an analysis of "install_requires" vs pip's + # requirements files see: + # https://packaging.python.org/en/latest/requirements.html + install_requires=[ + ], + + # List additional groups of dependencies here (e.g. development + # dependencies). You can install these using the following syntax, + # for example: + # $ pip install -e .[dev,test] + extras_require={ + 'dev': ['check-manifest', 'docutils', 'Pygments'], + # 'test': [ + # 'tox>=1.8.1', + # ], + # 'build_sphinx': [ + # 'sphinx', + # 'sphinxcontrib-napoleon', + # ], + }, + + # If there are data files included in your packages that need to be + # installed, specify them here. If using Python 2.6 or less, then these + # have to be included in MANIFEST.in as well. + # package_data={ + # 'tests': ['testrsa.key'], + # }, + + # To provide executable scripts, use entry points in preference to the + # "scripts" keyword. Entry points provide cross-platform support and allow + # pip to create the appropriate form of executable for the target platform. + # entry_points={ + # 'console_scripts': [ + # 'sshtunnel=sshtunnel:_cli_main', + # ] + # }, + + # Integrate tox with setuptools + # cmdclass={'test': Tox}, +) \ No newline at end of file diff --git a/testsuit/list-directory/file1 b/testsuit/list-directory/file1 new file mode 100644 index 0000000..c0d0fb4 --- /dev/null +++ b/testsuit/list-directory/file1 @@ -0,0 +1,2 @@ +line1 +line2 diff --git a/testsuit/list-directory/file2.txt b/testsuit/list-directory/file2.txt new file mode 100644 index 0000000..e6697bc --- /dev/null +++ b/testsuit/list-directory/file2.txt @@ -0,0 +1,6 @@ +This utility was born from the fact that I keep forgetting how to use "sed", +and I suck at Perl. It brings ad-hoc command-line piping sensibilities +to the Python interpeter. + +(Version 1.2 does better outputting of list-like results, +thanks to Mark Eichin.) diff --git a/testsuit/list-directory/file3.py b/testsuit/list-directory/file3.py new file mode 100644 index 0000000..3a3ab14 --- /dev/null +++ b/testsuit/list-directory/file3.py @@ -0,0 +1,2 @@ +import sys +sys.exit(22) diff --git a/testsuit/list-directory/file4.sh b/testsuit/list-directory/file4.sh new file mode 100644 index 0000000..a8840bb --- /dev/null +++ b/testsuit/list-directory/file4.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exit 33 diff --git a/testsuit/nginx.log b/testsuit/nginx.log new file mode 100644 index 0000000..e7d39d4 --- /dev/null +++ b/testsuit/nginx.log @@ -0,0 +1,18 @@ +[19/Jul/2016:21:50:12 +0300] 184.73.237.85 [1/18089] uget=- uset=uuid=0C00A8C064768E57F54178C002700503 - rt="0.005" uht="0.005" urt="0.005" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOdmTAeEH1AwVwAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/" +[19/Jul/2016:21:50:21 +0300] 50.112.95.211 [1/18091] uget=- uset=uuid=0C00A8C06D768E57F54178C002710503 - rt="0.005" uht="0.005" urt="0.005" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOdm3AeEH1AwVxAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/" +[19/Jul/2016:21:52:09 +0300] 54.251.34.67 [1/18093] uget=- uset=uuid=0C00A8C0D9768E57F54178C002720503 - rt="0.007" uht="0.007" urt="0.007" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOdtnAeEH1AwVyAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/" +[19/Jul/2016:21:52:35 +0300] 54.248.250.232 [1/18095] uget=- uset=uuid=0C00A8C0F3768E57F54178C002730503 - rt="0.006" uht="0.006" urt="0.006" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOdvPAeEH1AwVzAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/" +[19/Jul/2016:21:30:21 +0300] 50.112.95.211 [1/17958] uget=- uset=uuid=0C00A8C0BD718E57F3414CC0029F0503 - rt="0.004" uht="0.004" urt="0.004" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOcb3ATEHzAwWfAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/" +[19/Jul/2016:21:30:22 +0300] 92.248.130.149 [5/17954] uget=uuid=0C00A8C0BF940F57526A38BB02200303 uset=- - rt="0.424" uht="0.396" urt="0.410" - status=200 - "GET /admin/moktoring/session/add/ HTTP/1.1" len=672 sent=42330/41540 - "https://o7x.examus.net/admin/moktoring/session/" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" set_cookie="csrftoken=06uZBa0KnNSSsTg0q0G4ifvzZwQSBJ7n; expires=Tue, 18-Jul-2017 18:30:22 GMT; Max-Age=31449600; Path=/" +[19/Jul/2016:21:30:23 +0300] 92.248.130.149 [6/17954] uget=uuid=0C00A8C0BF940F57526A38BB02200303 uset=- - rt="0.029" uht="0.026" urt="0.029" - status=200 - "GET /admin/jsi18n/ HTTP/1.1" len=559 sent=15167/14680 - "https://o7x.examus.net/admin/moktoring/session/add/" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" set_cookie="-" +[19/Jul/2016:21:30:23 +0300] 92.248.130.149 [3/17950] uget=uuid=0C00A8C0BF940F57526A38BB02200303 uset=- - rt="0.000" uht="-" urt="-" - status=200 - "GET /static/admin/img/icon-calendar.svg HTTP/1.1" len=604 sent=1612/1086 - "https://o7x.examus.net/static/admin/css/widgets.css" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" set_cookie="-" +[19/Jul/2016:21:30:23 +0300] 92.248.130.149 [7/17954] uget=uuid=0C00A8C0BF940F57526A38BB02200303 uset=- - rt="0.000" uht="-" urt="-" - status=200 - "GET /static/admin/img/icon-clock.svg HTTP/1.1" len=601 sent=1202/677 - "https://o7x.examus.net/static/admin/css/widgets.css" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" set_cookie="-" +[19/Jul/2016:21:32:00 +0300] 54.251.34.67 [1/17963] uget=- uset=uuid=0C00A8C020728E57F3414CC002A00503 - rt="0.008" uht="0.008" urt="0.008" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOciDATEHzAwWgAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/" +[19/Jul/2016:21:32:35 +0300] 54.248.250.232 [1/17965] uget=- uset=uuid=0C00A8C043728E57F3414CC002A10503 - rt="0.007" uht="0.007" urt="0.007" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOckPATEHzAwWhAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/" +[19/Jul/2016:21:32:38 +0300] 184.73.237.85 [1/17967] uget=- uset=uuid=0C00A8C046728E57F3414CC002A20503 - rt="0.005" uht="0.005" urt="0.005" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOckbATEHzAwWiAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/" +[19/Jul/2016:21:32:40 +0300] 54.247.188.179 [1/17969] uget=- uset=uuid=0C00A8C048728E57F3414CC002A30503 - rt="0.007" uht="0.007" urt="0.007" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOckjATEHzAwWjAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/" +[19/Jul/2016:21:32:51 +0300] 50.112.95.211 [1/17971] uget=- uset=uuid=0C00A8C053728E57F3414CC002A40503 - rt="0.005" uht="0.005" urt="0.005" - status=302 - "HEAD / HTTP/1.0" len=214 sent=581/0 - "-" "NewRelicPinger/1.0 (1336397)" set_cookie="uuid=wKgADFeOclPATEHzAwWkAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/" +[19/Jul/2016:21:33:42 +0300] 109.173.101.89 [1/17973] uget=- uset=uuid=0C00A8C086728E57F3414CC002A50503 - rt="0.009" uht="0.005" urt="0.009" - status=302 - "GET /logout/?reason=startApplication HTTP/1.1" len=433 sent=606/5 - "-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" set_cookie="uuid=wKgADFeOcobATEHzAwWlAg==; expires=Thu, 31-Dec-37 23:55:55 GMT; path=/" +[19/Jul/2016:21:33:42 +0300] 109.173.101.89 [2/17973] uget=uuid=0C00A8C086728E57F3414CC002A50503 uset=- - rt="0.010" uht="0.006" urt="0.010" - status=302 - "GET / HTTP/1.1" len=442 sent=530/5 - "-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" set_cookie="-" +[19/Jul/2016:21:33:42 +0300] 109.173.101.89 [3/17973] uget=uuid=0C00A8C086728E57F3414CC002A50503 uset=- - rt="0.047" uht="0.039" urt="0.047" - status=200 - "GET /login/?next=/ HTTP/1.1" len=455 sent=8370/7580 - "-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" set_cookie="csrftoken=xZxfdkNjadcYIQXcKg8Wv0x8nQq9NwL0; expires=Tue, 18-Jul-2017 18:33:42 GMT; Max-Age=31449600; Path=/" +[19/Jul/2016:20:09:50 +0300] 92.248.130.149 [22/17536] uget=uuid=0C00A8C0BF940F57526A38BB02200303 uset=- - rt="0.078" uht="0.072" urt="0.078" - status=302 - "POST /admin/customauth/user/?q=%D0%9F%D0%B0%D1%81%D0%B5%D1%87%D0%BD%D0%B8%D0%BA HTTP/1.1" len=1189 sent=1063/5 - "https://o7x.examus.net/admin/customauth/user/?q=%D0%9F%D0%B0%D1%81%D0%B5%D1%87%D0%BD%D0%B8%D0%BA" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" set_cookie="messages=\x2292939612ff14300f94f8722c99618e22187e3503$[[\x5C\x22__json_message\x5C\x22\x5C0540\x5C05425\x5C054\x5C\x221 \x5C\x5Cu041f\x5C\x5Cu043e\x5C\x5Cu043b\x5C\x5Cu044c\x5C\x5Cu0437\x5C\x5Cu043e\x5C\x5Cu0432\x5C\x5Cu0430\x5C\x5Cu0442\x5C\x5Cu0435\x5C\x5Cu043b\x5C\x5Cu044c \x5C\x5Cu0431\x5C\x5Cu044b\x5C\x5Cu043b \x5C\x5Cu0443\x5C\x5Cu0441\x5C\x5Cu043f\x5C\x5Cu0435\x5C\x5Cu0448\x5C\x5Cu043d\x5C\x5Cu043e \x5C\x5Cu0438\x5C\x5Cu0437\x5C\x5Cu043c\x5C\x5Cu0435\x5C\x5Cu043d\x5C\x5Cu0435\x5C\x5Cu043d.\x5C\x22]]\x22; Path=/" diff --git a/testsuit/test.text b/testsuit/test.text new file mode 100644 index 0000000..aa27788 --- /dev/null +++ b/testsuit/test.text @@ -0,0 +1,8 @@ +This is my cat, + whose name is Betty. +This is my dog, + whose name is Frank. +This is my fish, + whose name is George. +This is my goat, + whose name is Adam. diff --git a/update.sh b/update.sh new file mode 100755 index 0000000..4d7539e --- /dev/null +++ b/update.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -x + +let num_git_changes="$(git status --porcelain | wc -l)" +let num_git_changes="$(printf '%s\n' "$num_git_changes")" +if [ "$num_git_changes" != "0" ]; then + echo "please commit or stash changes before" + exit 1 +fi + +python setup.py build sdist upload +if [ "$?" != "0" ]; then + echo "build and upload problem" + exit 2 +fi + +VERSION=$(python setup.py --version) +NAME=$(python setup.py --name) +NEXTVERSION=`echo $VERSION | python3 -c "v = input().strip().split('.'); v[-1] = str(int(v[-1]) + 1); print('.'.join(v))"` + +git tag -a v$VERSION -m "version $VERSION" +sed -e "s|$VERSION|$NEXTVERSION|g" -i.back $NAME.py $NAME/* +rm *.back