From 3639e92317d876eacb89afe3a4251003f211607c Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 17 Jan 2024 15:28:00 +0100 Subject: [PATCH 1/9] pytype flake8 mypy --- .gitignore | 1 + CONTRIBUTING.md | 2 +- dotdrop/action.py | 7 ++++--- dotdrop/cfg_yaml.py | 2 +- dotdrop/dictparser.py | 6 +++--- dotdrop/utils.py | 10 ++++++---- scripts/check-syntax.sh | 42 ++++++++++++++++++++++++++++++++++------- tests-requirements.txt | 7 +++++-- 8 files changed, 56 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index f7b26362c..f0184120d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ build/ tags env venv +.pytype # coverage stuff .coverage diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3822d9302..39df4c1e2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -209,7 +209,7 @@ for a match with the ignore patterns. Dotdrop is tested with the use of the [tests.sh](/tests.sh) script. -* Test for PEP8 compliance with `pylint`, `pycodestyle` and `pyflakes` (see [check-syntax.sh](/scripts/test-syntax.sh)) +* Test for PEP8 compliance with linters (see [check-syntax.sh](/scripts/test-syntax.sh)) * Test the documentation and links (see [check-doc.sh](/scripts/check-doc.sh)) * Run the unittests in [tests directory](/tests) with pytest (see [check-unittest.sh](/scripts/check-unittests.sh)) * Run the blackbox bash script tests in [tests-ng directory](/tests-ng) (see [check-tests-ng.sh](/scripts/check-tests-ng.sh)) diff --git a/dotdrop/action.py b/dotdrop/action.py index 6d032aaea..5f6079153 100644 --- a/dotdrop/action.py +++ b/dotdrop/action.py @@ -8,6 +8,7 @@ import subprocess import os +from typing import List # local imports from dotdrop.dictparser import DictParser @@ -17,7 +18,7 @@ class Cmd(DictParser): """A command to execute""" - args = [] + args: List[str] = [] eq_ignore = ('log',) descr = 'command' @@ -32,13 +33,13 @@ def __init__(self, key, action): self.silent = key.startswith('_') def _get_action(self, templater, debug): - action = None + action = '' try: action = templater.generate_string(self.action) except UndefinedException as exc: err = f'undefined variable for {self.descr}: \"{exc}\"' self.log.warn(err) - return False + return action if debug: self.log.dbg(f'{self.descr}:') self.log.dbg(f' - raw \"{self.action}\"') diff --git a/dotdrop/cfg_yaml.py b/dotdrop/cfg_yaml.py index 5d2c701cb..85218521b 100644 --- a/dotdrop/cfg_yaml.py +++ b/dotdrop/cfg_yaml.py @@ -29,7 +29,7 @@ try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore import tomli_w # local imports diff --git a/dotdrop/dictparser.py b/dotdrop/dictparser.py index 49017b810..9444b321c 100644 --- a/dotdrop/dictparser.py +++ b/dotdrop/dictparser.py @@ -27,9 +27,9 @@ def parse(cls, key, value): except AttributeError: pass newv = cls._adjust_yaml_keys(tmp) - if not key: - return cls(**newv) - return cls(key=key, **newv) + if key: + newv[key] = key + return cls(**newv) @classmethod def parse_dict(cls, items): diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 05c6f27fe..79bae696e 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -473,7 +473,11 @@ def get_module_from_path(path): importlib.machinery.SOURCE_SUFFIXES.append('') # import module spec = importlib.util.spec_from_file_location(module_name, path) + if not spec: + return None mod = importlib.util.module_from_spec(spec) + if not mod: + return None spec.loader.exec_module(mod) return mod @@ -507,8 +511,7 @@ def dependencies_met(): name = 'docopt' err = f'missing python module \"{name}\"' try: - from docopt import docopt - assert docopt + from docopt import docopt # noqa # pylint: disable=W0611 except ImportError as exc: raise UnmetDependency(err) from exc @@ -525,8 +528,7 @@ def dependencies_met(): name = 'ruamel.yaml' err = f'missing python module \"{name}\"' try: - from ruamel.yaml import YAML - assert YAML + from ruamel.yaml import YAML # noqa # pylint: disable=W0611 except ImportError as exc: raise UnmetDependency(err) from exc diff --git a/scripts/check-syntax.sh b/scripts/check-syntax.sh index 019620c11..4aaa3f668 100755 --- a/scripts/check-syntax.sh +++ b/scripts/check-syntax.sh @@ -28,12 +28,26 @@ fi echo "=> pycodestyle version:" pycodestyle --version -if ! which pyflakes >/dev/null 2>&1; then - echo "Install pyflakes" +if ! which flake8 >/dev/null 2>&1; then + echo "Install flake8" exit 1 fi -echo "=> pyflakes version:" -pyflakes --version +echo "=> flake8 version:" +flake8 --version + +if ! which pytype >/dev/null 2>&1; then + echo "Install pytype" + exit 1 +fi +echo "=> pytype version:" +pytype --version + +if ! which mypy >/dev/null 2>&1; then + echo "Install mypy" + exit 1 +fi +echo "=> mypy version:" +mypy --version # checking for TODO/FIXME echo "--------------------------------------" @@ -62,10 +76,10 @@ echo "checking dotdrop with pycodestyle" pycodestyle --ignore=W503,W504 dotdrop/ pycodestyle scripts/ -# pyflakes tests +# flake8 tests echo "------------------------------" -echo "checking dotdrop with pyflakes" -pyflakes dotdrop/ +echo "checking dotdrop with flake8" +flake8 dotdrop/ # pylint echo "----------------------------" @@ -90,6 +104,20 @@ pylint \ --disable=R0904 \ dotdrop/ +# pytype +echo "----------------------------" +echo "checking dotdrop with pytype" +pytype dotdrop/ + +# mypy +echo "----------------------------" +echo "checking dotdrop with mypy" +# --strict +mypy \ + --ignore-missing-imports \ + --allow-redefinition \ + dotdrop/ + # check shell scripts # SC2002: Useless cat # SC2126: Consider using grep -c instead of grep|wc -l diff --git a/tests-requirements.txt b/tests-requirements.txt index d496cecac..b5cb37eed 100644 --- a/tests-requirements.txt +++ b/tests-requirements.txt @@ -2,8 +2,11 @@ pycodestyle; python_version > '3.5' pytest; python_version > '3.5' coverage; python_version > '3.5' coveralls; python_version > '3.5' -pyflakes; python_version > '3.5' +flake8; python_version > '3.5' pylint; python_version > '3.5' halo; python_version > '3.5' distro; python_version > '3.5' -urllib3; python_version > '3.5' \ No newline at end of file +urllib3; python_version > '3.5' +pytype; python_version > '3.0' +mypy; python_version > '3.0' +types-requests; python_version > '3.0' \ No newline at end of file From d9fabe3e81b50fcd227fc14c1f32d533297cf139 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 17 Jan 2024 15:52:06 +0100 Subject: [PATCH 2/9] mypy strict --- dotdrop/action.py | 41 ++++++++++++++++++++++++++--------------- dotdrop/logger.py | 33 ++++++++++++++++++++------------- scripts/check-syntax.sh | 12 ++++++------ 3 files changed, 52 insertions(+), 34 deletions(-) diff --git a/dotdrop/action.py b/dotdrop/action.py index 5f6079153..07f1135f7 100644 --- a/dotdrop/action.py +++ b/dotdrop/action.py @@ -8,11 +8,12 @@ import subprocess import os -from typing import List +from typing import List, Dict, TypeVar, Optional # local imports from dotdrop.dictparser import DictParser from dotdrop.exceptions import UndefinedException +from dotdrop.templategen import Templategen class Cmd(DictParser): @@ -22,7 +23,7 @@ class Cmd(DictParser): eq_ignore = ('log',) descr = 'command' - def __init__(self, key, action): + def __init__(self, key: str, action: str): """constructor @key: action key @action: action string @@ -32,7 +33,7 @@ def __init__(self, key, action): self.action = action self.silent = key.startswith('_') - def _get_action(self, templater, debug): + def _get_action(self, templater: Templategen, debug: bool) -> str: action = '' try: action = templater.generate_string(self.action) @@ -46,7 +47,7 @@ def _get_action(self, templater, debug): self.log.dbg(f' - templated \"{action}\"') return action - def _get_args(self, templater): + def _get_args(self, templater: Templategen) -> List[str]: args = [] if not self.args: return args @@ -60,7 +61,9 @@ def _get_args(self, templater): return False return args - def execute(self, templater=None, debug=False): + def execute(self, + templater: Optional[Templategen] = None, + debug: bool = False) -> bool: """execute the command in the shell""" ret = 1 action = self.action @@ -101,13 +104,16 @@ def execute(self, templater=None, debug=False): return ret == 0 @classmethod - def _adjust_yaml_keys(cls, value): + def _adjust_yaml_keys(cls, value: str) -> Dict[str, str]: return {'action': value} - def __str__(self): + def __str__(self) -> str: return f'key:{self.key} -> \"{self.action}\"' +ActionT = TypeVar('ActionT', bound='Action') + + class Action(Cmd): """An action to execute""" @@ -115,7 +121,7 @@ class Action(Cmd): post = 'post' descr = 'action' - def __init__(self, key, kind, action): + def __init__(self, key: str, kind: str, action: str): """constructor @key: action key @kind: type of action (pre or post) @@ -125,33 +131,36 @@ def __init__(self, key, kind, action): self.kind = kind self.args = [] - def copy(self, args): + def copy(self, args: List[str]): """return a copy of this object with arguments""" action = Action(self.key, self.kind, self.action) action.args = args return action @classmethod - def parse(cls, key, value): + def parse(cls, key: str, value: str) -> ActionT: """parse key value into object""" val = {} val['kind'], val['action'] = value return cls(key=key, **val) - def __str__(self): + def __str__(self) -> str: out = f'{self.key}: [{self.kind}] \"{self.action}\"' return out - def __repr__(self): + def __repr__(self) -> str: return f'action({self.__str__()})' +TransformT = TypeVar('TransformT', bound='Transform') + + class Transform(Cmd): """A transformation on a dotfile""" descr = 'transformation' - def __init__(self, key, action): + def __init__(self, key: str, action: str): """constructor @key: action key @trans: action string @@ -159,13 +168,15 @@ def __init__(self, key, action): super().__init__(key, action) self.args = [] - def copy(self, args): + def copy(self, args: List[str]) -> TransformT: """return a copy of this object with arguments""" trans = Transform(self.key, self.action) trans.args = args return trans - def transform(self, arg0, arg1, templater=None, debug=False): + def transform(self, arg0: str, arg1: str, + templater: Optional[Templategen] = None, + debug: bool = False) -> bool: """ execute transformation with {0} and {1} where {0} is the file to transform diff --git a/dotdrop/logger.py b/dotdrop/logger.py index 03f9b9134..72711329d 100644 --- a/dotdrop/logger.py +++ b/dotdrop/logger.py @@ -22,10 +22,12 @@ class Logger: EMPH = '\033[33m' BOLD = '\033[1m' - def __init__(self, debug=False): + def __init__(self, debug: bool = False): self.debug = debug - def log(self, string, end='\n', pre='', bold=False): + def log(self, string: str, + end: str = '\n', pre: str = '', + bold: bool = False) -> None: """normal log""" cstart = self._color(self.BLUE) cend = self._color(self.RESET) @@ -37,13 +39,14 @@ def log(self, string, end='\n', pre='', bold=False): fmt = f'{pre}{cstart}{string}{end}{cend}' sys.stdout.write(fmt) - def sub(self, string, end='\n'): + def sub(self, string: str, + end: str = '\n') -> None: """sub log""" cstart = self._color(self.BLUE) cend = self._color(self.RESET) sys.stdout.write(f'\t{cstart}->{cend} {string}{end}') - def emph(self, string, stdout=True): + def emph(self, string: str, stdout: bool = True) -> None: """emphasis log""" cstart = self._color(self.EMPH) cend = self._color(self.RESET) @@ -53,46 +56,50 @@ def emph(self, string, stdout=True): else: sys.stdout.write(content) - def err(self, string, end='\n'): + def err(self, string: str, end: str = '\n') -> None: """error log""" cstart = self._color(self.RED) cend = self._color(self.RESET) msg = f'{string} {end}' sys.stderr.write(f'{cstart}[ERR] {msg}{cend}') - def warn(self, string, end='\n'): + def warn(self, string: str, end: str = '\n') -> None: """warning log""" cstart = self._color(self.YELLOW) cend = self._color(self.RESET) sys.stderr.write(f'{cstart}[WARN] {string} {end}{cend}') - def dbg(self, string, force=False): + def dbg(self, string: str, force: bool = False) -> None: """debug log""" if not force and not self.debug: return frame = inspect.stack()[1] - mod = inspect.getmodule(frame[0]).__name__ + + mod = inspect.getmodule(frame[0]) + mod_name = 'module?' + if mod: + mod_name = mod.__name__ func = inspect.stack()[1][3] cstart = self._color(self.MAGENTA) cend = self._color(self.RESET) clight = self._color(self.LMAGENTA) bold = self._color(self.BOLD) - line = f'{bold}{clight}[DEBUG][{mod}.{func}]' + line = f'{bold}{clight}[DEBUG][{mod_name}.{func}]' line += f'{cend}{cstart} {string}{cend}\n' sys.stderr.write(line) - def dry(self, string, end='\n'): + def dry(self, string: str, end: str = '\n') -> None: """dry run log""" cstart = self._color(self.GREEN) cend = self._color(self.RESET) sys.stdout.write(f'{cstart}[DRY] {string} {end}{cend}') @classmethod - def raw(cls, string, end='\n'): + def raw(cls, string: str, end: str = '\n') -> None: """raw log""" sys.stdout.write(f'{string}{end}') - def ask(self, query): + def ask(self, query: str) -> bool: """ask user for confirmation""" cstart = self._color(self.BLUE) cend = self._color(self.RESET) @@ -102,7 +109,7 @@ def ask(self, query): return resp == 'y' @classmethod - def _color(cls, col): + def _color(cls, col: str) -> str: """is color supported""" if not sys.stdout.isatty(): return '' diff --git a/scripts/check-syntax.sh b/scripts/check-syntax.sh index 4aaa3f668..f2dbbc275 100755 --- a/scripts/check-syntax.sh +++ b/scripts/check-syntax.sh @@ -104,20 +104,20 @@ pylint \ --disable=R0904 \ dotdrop/ -# pytype -echo "----------------------------" -echo "checking dotdrop with pytype" -pytype dotdrop/ - # mypy echo "----------------------------" echo "checking dotdrop with mypy" -# --strict mypy \ + --strict \ --ignore-missing-imports \ --allow-redefinition \ dotdrop/ +# pytype +echo "----------------------------" +echo "checking dotdrop with pytype" +pytype dotdrop/ + # check shell scripts # SC2002: Useless cat # SC2126: Consider using grep -c instead of grep|wc -l From 6dba75ff47c4435e36356ebc02c434ae7093b81d Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 18 Jan 2024 21:46:19 +0100 Subject: [PATCH 3/9] type utils --- dotdrop/utils.py | 109 ++++++++++++++++++++++++++++++----------------- 1 file changed, 69 insertions(+), 40 deletions(-) diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 79bae696e..59f2d622c 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -24,6 +24,12 @@ from dotdrop.logger import Logger from dotdrop.exceptions import UnmetDependency from dotdrop.version import __version__ as VERSION +from dotdrop.action import Action +from dotdrop.dotfile import Dotfile +from dotdrop.options import Options +from dotdrop.profile import Profile +from ruamel.yaml.comments import CommentedSeq +from typing import Any, Callable, List, Optional, Tuple, Union LOG = Logger() STAR = '*' @@ -40,7 +46,7 @@ NOREMOVE = [os.path.normpath(p) for p in DONOTDELETE] -def run(cmd, debug=False): +def run(cmd: List[str], debug: bool=False) -> Tuple[bool, str]: """run a command (expects a list)""" if debug: fcmd = ' '.join(cmd) @@ -55,7 +61,7 @@ def run(cmd, debug=False): return ret == 0, lines -def write_to_tmpfile(content): +def write_to_tmpfile(content: bytes) -> str: """write some content to a tmp file""" path = get_tmpfile() with open(path, 'wb') as file: @@ -63,7 +69,7 @@ def write_to_tmpfile(content): return path -def shellrun(cmd, debug=False): +def shellrun(cmd: str, debug: bool=False) -> Tuple[bool, str]: """ run a command in the shell (expects a string) returns True|False, output @@ -76,7 +82,7 @@ def shellrun(cmd, debug=False): return ret == 0, out -def userinput(prompt, debug=False): +def userinput(prompt: str, debug: bool = False) -> str: """ get user input return user input @@ -90,13 +96,13 @@ def userinput(prompt, debug=False): return res -def fastdiff(left, right): +def fastdiff(left: str, right: str) -> bool: """fast compare files and returns True if different""" return not filecmp.cmp(left, right, shallow=False) -def diff(original, modified, - diff_cmd='', debug=False): +def diff(original: str, modified: str, + diff_cmd: str='', debug: bool=False) -> str: """compare two files, returns '' if same""" if not diff_cmd: diff_cmd = 'diff -r -u {0} {1}' @@ -112,7 +118,7 @@ def diff(original, modified, return out -def get_tmpdir(): +def get_tmpdir() -> str: """create and return the temporary directory""" # pylint: disable=W0603 global TMPDIR @@ -124,7 +130,7 @@ def get_tmpdir(): return tmp -def _get_tmpdir(): +def _get_tmpdir() -> str: """create the tmpdir""" try: if ENV_TEMP in os.environ: @@ -139,21 +145,21 @@ def _get_tmpdir(): return tempfile.mkdtemp(prefix='dotdrop-') -def get_tmpfile(): +def get_tmpfile() -> str: """create a temporary file""" tmpdir = get_tmpdir() return tempfile.NamedTemporaryFile(prefix='dotdrop-', dir=tmpdir, delete=False).name -def get_unique_tmp_name(): +def get_unique_tmp_name() -> str: """get a unique file name (not created)""" unique = str(uuid.uuid4()) tmpdir = get_tmpdir() return os.path.join(tmpdir, unique) -def removepath(path, logger=None): +def removepath(path: str, logger: Optional[Logger]=None) -> None: """ remove a file/directory/symlink if logger is defined, OSError are catched @@ -193,7 +199,7 @@ def removepath(path, logger=None): raise OSError(err) from exc -def samefile(path1, path2): +def samefile(path1: str, path2: str) -> bool: """return True if represent the same file""" if not os.path.exists(path1): return False @@ -202,12 +208,12 @@ def samefile(path1, path2): return os.path.samefile(path1, path2) -def header(): +def header() -> str: """return dotdrop header""" return 'This dotfile is managed using dotdrop' -def content_empty(string): +def content_empty(string: bytes) -> bool: """return True if is empty or only one CRLF""" if not string: return True @@ -216,7 +222,7 @@ def content_empty(string): return False -def strip_home(path): +def strip_home(path: str) -> str: """properly strip $HOME from path""" home = os.path.expanduser('~') + os.sep if path.startswith(home): @@ -224,7 +230,7 @@ def strip_home(path): return path -def _match_ignore_pattern(path, pattern, debug=False): +def _match_ignore_pattern(path: str, pattern: str, debug: bool=False) -> bool: """ returns true if path matches the pattern we test the entire path but also @@ -249,7 +255,10 @@ def _match_ignore_pattern(path, pattern, debug=False): return False -def _must_ignore(path, ignores, neg_ignores, debug=False): +def _must_ignore(path: str, + ignores: List[str], + neg_ignores: List[str], + debug: bool=False) -> bool: """ return true if path matches any ignore patterns """ @@ -315,7 +324,9 @@ def _must_ignore(path, ignores, neg_ignores, debug=False): return True -def must_ignore(paths, ignores, debug=False): +def must_ignore(paths: List[str], + ignores: List[str], + debug: bool=False) -> bool: """ return true if any paths in list matches any ignore patterns """ @@ -336,7 +347,10 @@ def must_ignore(paths, ignores, debug=False): return False -def _cp(src, dst, ignore_func=None, debug=False): +def _cp(src: str, + dst: str, + ignore_func: Optional[Callable]=None, + debug: bool=False) -> int: """ the copy function for copytree returns the numb of files copied @@ -363,7 +377,7 @@ def _cp(src, dst, ignore_func=None, debug=False): return 0 -def copyfile(src, dst, debug=False): +def copyfile(src: str, dst: str, debug: bool=False) -> bool: """ copy file from src to dst no dir expected! @@ -372,7 +386,10 @@ def copyfile(src, dst, debug=False): return _cp(src, dst, debug=debug) == 1 -def copytree_with_ign(src, dst, ignore_func=None, debug=False): +def copytree_with_ign(src: str, + dst: str, + ignore_func: Optional[Callable]=None, + debug: bool=False) -> int: """ copytree with support for ignore returns the numb of files installed @@ -407,7 +424,7 @@ def copytree_with_ign(src, dst, ignore_func=None, debug=False): return copied_count -def uniq_list(a_list): +def uniq_list(a_list: List[str]) -> List[str]: """unique elements of a list while preserving order""" new = [] if not a_list: @@ -418,7 +435,9 @@ def uniq_list(a_list): return new -def ignores_to_absolute(ignores, prefixes, debug=False): +def ignores_to_absolute(ignores: List[str], + prefixes: List[str], + debug: bool=False) -> List[str]: """allow relative ignore pattern""" new = [] LOG.dbg(f'ignores before patching: {ignores}', force=debug) @@ -464,7 +483,7 @@ def get_module_functions(mod): return funcs -def get_module_from_path(path): +def get_module_from_path(path: str): """get module from path""" if not path or not os.path.exists(path): return None @@ -482,7 +501,7 @@ def get_module_from_path(path): return mod -def dependencies_met(): +def dependencies_met() -> None: """make sure all dependencies are met""" # check unix tools deps # diff command is checked in settings.py @@ -563,7 +582,7 @@ def dependencies_met(): # pylint: enable=C0415 -def mirror_file_rights(src, dst): +def mirror_file_rights(src: str, dst: str) -> None: """mirror file rights of src to dst (can rise exc)""" if not os.path.exists(src) or not os.path.exists(dst): return @@ -571,7 +590,7 @@ def mirror_file_rights(src, dst): os.chmod(dst, rights) -def get_umask(): +def get_umask() -> int: """return current umask value""" cur = os.umask(0) os.umask(cur) @@ -579,7 +598,7 @@ def get_umask(): return cur -def get_default_file_perms(path, umask): +def get_default_file_perms(path: str, umask: int) -> int: """get default rights for a file""" base = 0o666 if os.path.isdir(path): @@ -587,14 +606,14 @@ def get_default_file_perms(path, umask): return base - umask -def get_file_perm(path): +def get_file_perm(path: str) -> int: """return file permission""" if not os.path.exists(path): return 0o777 return os.stat(path, follow_symlinks=True).st_mode & 0o777 -def chmod(path, mode, debug=False): +def chmod(path: str, mode: int, debug: bool=False) -> bool: """change mode of file""" if debug: LOG.dbg(f'chmod {mode:o} {path}', force=True) @@ -602,7 +621,8 @@ def chmod(path, mode, debug=False): return get_file_perm(path) == mode -def adapt_workers(options, logger): +def adapt_workers(options: Options, + logger: Logger) -> None: """adapt number of workers if safe/dry""" if options.safe and options.workers > 1: logger.warn('workers set to 1 when --force is not used') @@ -612,16 +632,20 @@ def adapt_workers(options, logger): options.workers = 1 -def categorize(function, iterable): - """separate an iterable into elements for which +def categorize(function: Callable, iterable: List[str]) -> List[str]: + """ + separate an iterable into elements for which function(element) is true for each element and for which function(element) is false for each - element""" + element + """ return (tuple(filter(function, iterable)), tuple(itertools.filterfalse(function, iterable))) -def debug_list(title, elems, debug): +def debug_list(title: str, + elems: List[Any], + debug: bool) -> None: """pretty print list""" if not debug: return @@ -630,7 +654,9 @@ def debug_list(title, elems, debug): LOG.dbg(f'\t- {elem}', force=debug) -def debug_dict(title, elems, debug): +def debug_dict(title: str, + elems: Any, + debug: bool) -> None: """pretty print dict""" if not debug: return @@ -644,7 +670,7 @@ def debug_dict(title, elems, debug): LOG.dbg(f'\t- \"{k}\": {val}', force=debug) -def check_version(): +def check_version() -> None: """ get dotdrop latest version on github compare with "version" @@ -677,7 +703,10 @@ def check_version(): LOG.warn(msg) -def pivot_path(path, newdir, striphome=False, logger=None): +def pivot_path(path: str, + newdir: str, + striphome: bool=False, + logger: Optional[Logger]=None) -> str: """change path to be under newdir""" if logger: logger.dbg(f'pivot new dir: \"{newdir}\"') @@ -691,7 +720,7 @@ def pivot_path(path, newdir, striphome=False, logger=None): return new -def is_bin_in_path(command): +def is_bin_in_path(command: str) -> bool: """ check binary from command is in path """ From a2fba6a906120fcce7173e97ae2056e33bc9429e Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 18 Jan 2024 22:07:21 +0100 Subject: [PATCH 4/9] utils typing --- dotdrop/utils.py | 49 +++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 59f2d622c..18e022f50 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -24,12 +24,9 @@ from dotdrop.logger import Logger from dotdrop.exceptions import UnmetDependency from dotdrop.version import __version__ as VERSION -from dotdrop.action import Action -from dotdrop.dotfile import Dotfile from dotdrop.options import Options -from dotdrop.profile import Profile -from ruamel.yaml.comments import CommentedSeq -from typing import Any, Callable, List, Optional, Tuple, Union +from typing import Any, Callable, List, \ + Optional, Tuple LOG = Logger() STAR = '*' @@ -56,8 +53,8 @@ def run(cmd: List[str], debug: bool=False) -> Tuple[bool, str]: stderr=subprocess.STDOUT) as proc: out, _ = proc.communicate() ret = proc.returncode - out = out.splitlines(keepends=True) - lines = ''.join([x.decode('utf-8', 'replace') for x in out]) + outlines = out.splitlines(keepends=True) + lines = ''.join([x.decode('utf-8', 'replace') for x in outlines]) return ret == 0, lines @@ -349,7 +346,7 @@ def must_ignore(paths: List[str], def _cp(src: str, dst: str, - ignore_func: Optional[Callable]=None, + ignore_func: Optional[Callable[[str], bool]]=None, debug: bool=False) -> int: """ the copy function for copytree @@ -388,7 +385,7 @@ def copyfile(src: str, dst: str, debug: bool=False) -> bool: def copytree_with_ign(src: str, dst: str, - ignore_func: Optional[Callable]=None, + ignore_func: Optional[Callable[[str], bool]]=None, debug: bool=False) -> int: """ copytree with support for ignore @@ -426,7 +423,7 @@ def copytree_with_ign(src: str, def uniq_list(a_list: List[str]) -> List[str]: """unique elements of a list while preserving order""" - new = [] + new: List[str] = [] if not a_list: return new for elem in a_list: @@ -472,7 +469,7 @@ def ignores_to_absolute(ignores: List[str], return new -def get_module_functions(mod): +def get_module_functions(mod): # type: ignore """return a list of fonction from a module""" funcs = [] for memb in inspect.getmembers(mod): @@ -483,7 +480,7 @@ def get_module_functions(mod): return funcs -def get_module_from_path(path: str): +def get_module_from_path(path: str): # type: ignore """get module from path""" if not path or not os.path.exists(path): return None @@ -491,10 +488,10 @@ def get_module_from_path(path: str): # allow any type of files importlib.machinery.SOURCE_SUFFIXES.append('') # import module - spec = importlib.util.spec_from_file_location(module_name, path) + spec = importlib.util.spec_from_file_location(module_name, path) # type: ignore if not spec: return None - mod = importlib.util.module_from_spec(spec) + mod = importlib.util.module_from_spec(spec) # type: ignore if not mod: return None spec.loader.exec_module(mod) @@ -519,7 +516,7 @@ def dependencies_met() -> None: name = 'python-magic' err = f'missing python module \"{name}\"' try: - import magic + import magic # type: ignore assert magic if not hasattr(magic, 'from_file'): LOG.warn(err) @@ -547,7 +544,7 @@ def dependencies_met() -> None: name = 'ruamel.yaml' err = f'missing python module \"{name}\"' try: - from ruamel.yaml import YAML # noqa # pylint: disable=W0611 + from ruamel.yaml import YAML # type: ignore # noqa # pylint: disable=W061 except ImportError as exc: raise UnmetDependency(err) from exc @@ -565,7 +562,7 @@ def dependencies_met() -> None: name = 'tomli_w' err = f'missing python module \"{name}\"' try: - import tomli_w + import tomli_w # type: ignore assert tomli_w except ImportError as exc: raise UnmetDependency(err) from exc @@ -574,7 +571,7 @@ def dependencies_met() -> None: name = 'distro' err = f'missing python module \"{name}\"' try: - import distro + import distro # type: ignore assert distro except ImportError as exc: raise UnmetDependency(err) from exc @@ -632,15 +629,15 @@ def adapt_workers(options: Options, options.workers = 1 -def categorize(function: Callable, iterable: List[str]) -> List[str]: +def categorize(function: Callable[[str],bool], + iterable: List[str]) -> Tuple[List[str], List[str]]: """ - separate an iterable into elements for which - function(element) is true for each element and - for which function(element) is false for each - element + separate an iterable into two lists: + - elements for which function(element) is true for each element + - elements for which function(element) is false for each element """ - return (tuple(filter(function, iterable)), - tuple(itertools.filterfalse(function, iterable))) + return list(filter(function, iterable)), \ + list(itertools.filterfalse(function, iterable)) def debug_list(title: str, @@ -724,7 +721,7 @@ def is_bin_in_path(command: str) -> bool: """ check binary from command is in path """ - bpath = "" + bpath: Optional[str | None] = "" if not command: return False try: From b9b4fc87f809daefbeda2be250aeb428b5e131ff Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 18 Jan 2024 22:10:06 +0100 Subject: [PATCH 5/9] restore dictparser --- dotdrop/dictparser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dotdrop/dictparser.py b/dotdrop/dictparser.py index 9444b321c..49017b810 100644 --- a/dotdrop/dictparser.py +++ b/dotdrop/dictparser.py @@ -27,9 +27,9 @@ def parse(cls, key, value): except AttributeError: pass newv = cls._adjust_yaml_keys(tmp) - if key: - newv[key] = key - return cls(**newv) + if not key: + return cls(**newv) + return cls(key=key, **newv) @classmethod def parse_dict(cls, items): From f85139d7c4d521181b2bcf0ccd0c983a5e590c2f Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 18 Jan 2024 22:10:13 +0100 Subject: [PATCH 6/9] typing --- dotdrop/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotdrop/importer.py b/dotdrop/importer.py index cd9388f51..aaa87fd8b 100644 --- a/dotdrop/importer.py +++ b/dotdrop/importer.py @@ -289,7 +289,7 @@ def _already_exists(self, src, dst): return True return False - def _ignore(self, path): + def _ignore(self, path: str): if must_ignore([path], self.ignore, debug=self.debug): self.log.dbg(f'ignoring import of {path}') self.log.warn(f'{path} ignored') From f3aa885d46ad9edcc630929e8308ccd472aab089 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 18 Jan 2024 22:17:06 +0100 Subject: [PATCH 7/9] utils typing --- dotdrop/utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dotdrop/utils.py b/dotdrop/utils.py index 18e022f50..fe940dcbb 100644 --- a/dotdrop/utils.py +++ b/dotdrop/utils.py @@ -19,14 +19,16 @@ import sys import requests from packaging import version +from typing import Any, Callable, List, \ + Optional, Tuple, TYPE_CHECKING # local import from dotdrop.logger import Logger from dotdrop.exceptions import UnmetDependency from dotdrop.version import __version__ as VERSION -from dotdrop.options import Options -from typing import Any, Callable, List, \ - Optional, Tuple +if TYPE_CHECKING: + from dotdrop.options import Options + LOG = Logger() STAR = '*' @@ -618,7 +620,7 @@ def chmod(path: str, mode: int, debug: bool=False) -> bool: return get_file_perm(path) == mode -def adapt_workers(options: Options, +def adapt_workers(options: "Options", logger: Logger) -> None: """adapt number of workers if safe/dry""" if options.safe and options.workers > 1: From 3a336dc8a8e5ea9be2711d5a6b9c18b4a24c4288 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 18 Jan 2024 22:24:16 +0100 Subject: [PATCH 8/9] options typing --- dotdrop/options.py | 55 +++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/dotdrop/options.py b/dotdrop/options.py index 3ead97145..11f94a51c 100644 --- a/dotdrop/options.py +++ b/dotdrop/options.py @@ -21,6 +21,8 @@ from dotdrop.action import Action from dotdrop.utils import uniq_list, debug_list, debug_dict from dotdrop.exceptions import YamlException, OptionsException +from typing import Any, Dict, Optional, List + ENV_PROFILE = 'DOTDROP_PROFILE' ENV_CONFIG = 'DOTDROP_CONFIG' @@ -112,26 +114,26 @@ class AttrMonitor: _set_attr_err = False # pylint: disable=W0235 - def __setattr__(self, key, value): + def __setattr__(self, key: str, value: Any) -> None: """monitor attribute setting""" super().__setattr__(key, value) # pylint: enable=W0235 - def _attr_set(self, attr): + def _attr_set(self, attr: str) -> None: """do something when unexistent attr is set""" class Options(AttrMonitor): """dotdrop options manager""" - def __init__(self, args=None): + def __init__(self, args: Optional[Dict[str,str]]=None) -> None: """constructor @args: argument dictionary (if None use sys) """ # attributes gotten from self.conf.get_settings() self.banner = None self.showdiff = None - self.default_actions = [] + self.default_actions: List[Action] = [] self.instignore = None self.force_chmod = None self.cmpignore = None @@ -159,6 +161,8 @@ def __init__(self, args=None): # selected profile self.profile = self.args['--profile'] self.confpath = self._get_config_path() + if not self.confpath: + raise YamlException('no config file found') self.confpath = os.path.abspath(self.confpath) self.log.dbg(f'config abs path: {self.confpath}') if not self.confpath: @@ -185,12 +189,12 @@ def __init__(self, args=None): # start monitoring for bad attribute self._set_attr_err = True - def debug_command(self): + def debug_command(self) -> None: """print the original command""" self.log.dbg(f'command: {self.argv}') @classmethod - def _get_config_from_env(cls, name): + def _get_config_from_env(cls, name: str) -> str: # look in XDG_CONFIG_HOME if ENV_XDG in os.environ: cfg = os.path.expanduser(os.environ[ENV_XDG]) @@ -200,7 +204,7 @@ def _get_config_from_env(cls, name): return '' @classmethod - def _get_config_from_fs(cls, name): + def _get_config_from_fs(cls, name: str) -> str: """get config from filesystem""" # look in ~/.config/dotdrop cfg = os.path.expanduser(HOMECFG) @@ -220,12 +224,12 @@ def _get_config_from_fs(cls, name): return '' - def _get_config_path(self): + def _get_config_path(self) -> str: """get the config path""" # cli provided if self.args['--cfg']: self.log.dbg(f'config from --cfg {self.args["--cfg"]}') - return os.path.expanduser(self.args['--cfg']) + return str(os.path.expanduser(self.args['--cfg'])) # environment variable provided if ENV_CONFIG in os.environ: @@ -263,14 +267,14 @@ def _get_config_path(self): return path self.log.dbg('no config file found') - return None + return '' - def _header(self): + def _header(self) -> None: """display the header""" self.log.log(BANNER) self.log.log('') - def _read_config(self): + def _read_config(self) -> None: """read the config file""" self.conf = CfgAggregator(self.confpath, self.profile, @@ -282,12 +286,12 @@ def _read_config(self): for k, val in settings.items(): setattr(self, k, val) - def _apply_args_files(self): + def _apply_args_files(self) -> None: """files specifics""" self.files_templateonly = self.args['--template'] self.files_grepable = self.args['--grepable'] - def _apply_args_install(self): + def _apply_args_install(self) -> None: """install specifics""" self.install_force_action = self.args['--force-actions'] self.install_temporary = self.args['--temp'] @@ -305,7 +309,7 @@ def _apply_args_install(self): self.clear_workdir self.install_remove_existing = self.args['--remove-existing'] - def _apply_args_compare(self): + def _apply_args_compare(self) -> None: """compare specifics""" self.compare_focus = self.args['--file'] self.compare_ignore = self.args['--ignore'] @@ -313,10 +317,11 @@ def _apply_args_compare(self): self.compare_ignore.append(f'*{self.install_backup_suffix}') self.compare_ignore = uniq_list(self.compare_ignore) self.compare_fileonly = self.args['--file-only'] + self.ignore_missing_in_dotdrop: bool = False self.ignore_missing_in_dotdrop = self.ignore_missing_in_dotdrop or \ self.args['--ignore-missing'] - def _apply_args_import(self): + def _apply_args_import(self) -> None: """import specifics""" self.import_path = self.args[''] self.import_as = self.args['--as'] @@ -328,7 +333,7 @@ def _apply_args_import(self): self.import_trans_install = self.args['--transr'] self.import_trans_update = self.args['--transw'] - def _apply_args_update(self): + def _apply_args_update(self) -> None: """update specifics""" self.update_path = self.args[''] self.update_iskey = self.args['--key'] @@ -338,24 +343,24 @@ def _apply_args_update(self): self.update_ignore = uniq_list(self.update_ignore) self.update_showpatch = self.args['--show-patch'] - def _apply_args_profiles(self): + def _apply_args_profiles(self) -> None: """profiles specifics""" self.profiles_grepable = self.args['--grepable'] - def _apply_args_remove(self): + def _apply_args_remove(self) -> None: """remove specifics""" self.remove_path = self.args[''] self.remove_iskey = self.args['--key'] - def _apply_args_uninstall(self): + def _apply_args_uninstall(self) -> None: """uninstall specifics""" self.uninstall_key = self.args[''] - def _apply_args_detail(self): + def _apply_args_detail(self) -> None: """detail specifics""" self.detail_keys = self.args[''] - def _apply_args(self): + def _apply_args(self) -> None: """apply cli args as attribute""" # the commands self.cmd_profiles = self.args['profiles'] @@ -418,7 +423,7 @@ def _apply_args(self): # "uninstall" specifics self._apply_args_uninstall() - def _fill_attr(self): + def _fill_attr(self) -> None: """create attributes from conf""" # defined variables self.variables = self.conf.get_variables() @@ -427,7 +432,7 @@ def _fill_attr(self): # all defined profiles self.profiles = self.conf.get_profiles() - def _debug_attr(self): + def _debug_attr(self) -> None: """debug display all of this class attributes""" if not self.debug: return @@ -445,6 +450,6 @@ def _debug_attr(self): else: self.log.dbg(f'-> {att}: {val}') - def _attr_set(self, attr): + def _attr_set(self, attr: str) -> None: """error when some inexistent attr is set""" raise OptionsException(f'bad option: {attr}') From 377b4770f6853559d74ab3a07a359be1871635f0 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 18 Jan 2024 22:26:36 +0100 Subject: [PATCH 9/9] action typing --- dotdrop/action.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dotdrop/action.py b/dotdrop/action.py index 07f1135f7..32e0c0d51 100644 --- a/dotdrop/action.py +++ b/dotdrop/action.py @@ -23,7 +23,7 @@ class Cmd(DictParser): eq_ignore = ('log',) descr = 'command' - def __init__(self, key: str, action: str): + def __init__(self, key: str, action: str) -> None: """constructor @key: action key @action: action string @@ -121,7 +121,7 @@ class Action(Cmd): post = 'post' descr = 'action' - def __init__(self, key: str, kind: str, action: str): + def __init__(self, key: str, kind: str, action: str) -> None: """constructor @key: action key @kind: type of action (pre or post) @@ -131,7 +131,7 @@ def __init__(self, key: str, kind: str, action: str): self.kind = kind self.args = [] - def copy(self, args: List[str]): + def copy(self, args: List[str]) -> "Action": """return a copy of this object with arguments""" action = Action(self.key, self.kind, self.action) action.args = args