tools(//tools/linter): Adding linters for python and C++

narendasan · narendasan · commit 4be0351ab2ca · 2020-10-30T17:12:52.000-07:00
Signed-off-by: Naren Dasan &lt;naren@narendasan.com&gt;
Signed-off-by: Naren Dasan &lt;narens@nvidia.com&gt;
diff --git a/.clang-format b/.clang-format
@@ -0,0 +1,87 @@
+---
+AccessModifierOffset: -1
+AlignAfterOpenBracket: AlwaysBreak
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlinesLeft: true
+AlignOperands:   false
+AlignTrailingComments: false
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Empty
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: false
+BinPackParameters: false
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: false
+ColumnLimit:     120
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ForEachMacros:   [ FOR_EACH_RANGE, FOR_EACH, ]
+IncludeCategories:
+  - Regex:           '^<.*\.h(pp)?>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IndentCaseLabels: true
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 2000000
+PointerAlignment: Left
+ReflowComments:  true
+SortIncludes:    true
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Cpp11
+TabWidth:        8
+UseTab:          Never
diff --git a/.clang-tidy b/.clang-tidy
@@ -0,0 +1,36 @@
+---
+# NOTE there must be no spaces before the '-', so put the comma last.
+InheritParentConfig: true
+Checks: '
+bugprone-*,
+-bugprone-forward-declaration-namespace,
+-bugprone-macro-parentheses,
+-bugprone-lambda-function-name,
+cppcoreguidelines-*,
+-cppcoreguidelines-interfaces-global-init,
+-cppcoreguidelines-owning-memory,
+-cppcoreguidelines-pro-bounds-array-to-pointer-decay,
+-cppcoreguidelines-pro-bounds-constant-array-index,
+-cppcoreguidelines-pro-bounds-pointer-arithmetic,
+-cppcoreguidelines-pro-type-cstyle-cast,
+-cppcoreguidelines-pro-type-reinterpret-cast,
+-cppcoreguidelines-pro-type-static-cast-downcast,
+-cppcoreguidelines-pro-type-union-access,
+-cppcoreguidelines-pro-type-vararg,
+-cppcoreguidelines-special-member-functions,
+-facebook-hte-RelativeInclude,
+hicpp-exception-baseclass,
+hicpp-avoid-goto,
+modernize-*,
+-modernize-return-braced-init-list,
+-modernize-use-auto,
+-modernize-use-default-member-init,
+-modernize-use-using,
+-modernize-use-trailing-return-type,
+performance-*,
+-performance-noexcept-move-constructor,
+'
+HeaderFilterRegex: 'torch/csrc/.*'
+AnalyzeTemporaryDtors: false
+CheckOptions:
+...
diff --git a/.style.yapf b/.style.yapf
@@ -0,0 +1,8 @@
+[style]
+based_on_style = google
+indent_width = 4
+COLUMN_LIMIT = 120
+SPACES_BEFORE_COMMENT = 2
+EACH_DICT_ENTRY_ON_SEPARATE_LINE = 1
+SPLIT_BEFORE_FIRST_ARGUMENT = false
+SPLIT_BEFORE_NAMED_ASSIGNS = true
diff --git a/WORKSPACE b/WORKSPACE
@@ -139,3 +139,12 @@ pip3_import(
 load("@py_test_deps//:requirements.bzl", "pip_install")
 pip_install()
 
+pip3_import(
+   name = "pylinter_deps",
+   requirements = "//tools/linter:requirements.txt",
+)
+
+load("@pylinter_deps//:requirements.bzl", "pip_install")
+pip_install()
+
+
diff --git a/tools/BUILD b/tools/BUILD
diff --git a/tools/linter/BUILD b/tools/linter/BUILD
@@ -0,0 +1,49 @@
+load("@pylinter_deps//:requirements.bzl", "requirement")
+
+py_binary(
+    name = "cpplint",
+    srcs = ["cpplint.py"],
+    python_version = "PY3",
+    srcs_version = "PY3",
+    visibility = ["//visibility:public"],
+    deps = [":utils"],
+)
+
+py_binary(
+    name = "cpplint_diff",
+    srcs = ["cpplint_diff.py"],
+    python_version = "PY3",
+    srcs_version = "PY3",
+    visibility = ["//visibility:public"],
+    deps = [":utils"],
+)
+
+py_binary(
+    name = "pylint",
+    srcs = ["pylint.py"],
+    python_version = "PY3",
+    srcs_version = "PY3",
+    visibility = ["//visibility:public"],
+    deps = [
+        ":utils",
+        requirement("yapf"),
+    ],
+)
+
+py_binary(
+    name = "pylint_diff",
+    srcs = ["pylint_diff.py"],
+    python_version = "PY3",
+    srcs_version = "PY3",
+    visibility = ["//visibility:public"],
+    deps = [
+        ":utils",
+        requirement("yapf"),
+    ],
+)
+
+py_library(
+    name = "utils",
+    srcs = ["utils.py"],
+    srcs_version = "PY3",
+)
diff --git a/tools/linter/cpplint.py b/tools/linter/cpplint.py
@@ -0,0 +1,38 @@
+import os
+import sys
+import glob
+import subprocess
+import utils
+
+VALID_CPP_FILE_TYPES = [".cpp", ".cc", ".c", ".cu", ".hpp", ".h", ".cuh"]
+
+
+def lint(user, target_files, change_file=True):
+    cmd = ['clang-format']
+    if change_file:
+        cmd.append("-i")
+        print(
+            "\033[93mWARNING:\033[0m This command is modifying your files with the recommended linting, you should review the changes before committing"
+        )
+    for f in target_files:
+        cmd.append(f)
+        subprocess.run(cmd)
+        subprocess.run(["chown", user + ":" + user, f])
+        subprocess.run(["chmod", "u+rw,g+rw", f])
+
+
+if __name__ == "__main__":
+    BAZEL_ROOT = utils.find_bazel_root()
+    USER = BAZEL_ROOT.split('/')[2]
+    projects = utils.CHECK_PROJECTS(sys.argv[1:])
+    if "//..." in projects:
+        projects = [p.replace(BAZEL_ROOT, "/")[:-1] for p in glob.glob(BAZEL_ROOT + '/*/')]
+        projects = [p for p in projects if p not in utils.BLACKLISTED_BAZEL_TARGETS]
+
+    for p in projects:
+        if p.endswith("/..."):
+            p = p[:-4]
+        path = BAZEL_ROOT + '/' + p[2:]
+        files = utils.glob_files(path, VALID_CPP_FILE_TYPES)
+        if files != []:
+            lint(USER, files)
diff --git a/tools/linter/cpplint_diff.py b/tools/linter/cpplint_diff.py
@@ -0,0 +1,38 @@
+import os
+import sys
+import glob
+import subprocess
+import utils
+
+VALID_CPP_FILE_TYPES = [".cpp", ".cc", ".c", ".cu", ".hpp", ".h", ".cuh"]
+
+
+def lint(target_files):
+    failure = False
+    for f in target_files:
+        with open("/tmp/changes.txt", "w") as changes:
+            subprocess.run(['clang-format', f], stdout=changes)
+        output = subprocess.run(["git", "diff", "-u", "--color", f, "/tmp/changes.txt"])
+        if output.returncode != 0:
+            failure = True
+    return failure
+
+
+if __name__ == "__main__":
+    BAZEL_ROOT = utils.find_bazel_root()
+    projects = utils.CHECK_PROJECTS(sys.argv[1:])
+    if "//..." in projects:
+        projects = [p.replace(BAZEL_ROOT, "/")[:-1] for p in glob.glob(BAZEL_ROOT + '/*/')]
+        projects = [p for p in projects if p not in utils.BLACKLISTED_BAZEL_TARGETS]
+
+    failure = False
+    for p in projects:
+        if p.endswith("/..."):
+            p = p[:-4]
+        path = BAZEL_ROOT + '/' + p[2:]
+        files = utils.glob_files(path, VALID_CPP_FILE_TYPES)
+        if files != []:
+            failure = lint(files)
+    if failure:
+        print("\033[91mERROR:\033[0m Some files do not conform to style guidelines")
+        sys.exit(1)
diff --git a/tools/linter/pylint.py b/tools/linter/pylint.py
@@ -0,0 +1,44 @@
+import os
+import sys
+import glob
+import utils
+import subprocess
+import yapf
+
+VALID_PY_FILE_TYPES = [".py"]
+
+
+def lint(user, target_files, conf, change_file=True):
+    return yapf.FormatFiles(
+        filenames=target_files,
+        lines=None,
+        style_config=conf,
+        no_local_style=None,
+        in_place=change_file,
+        print_diff=False,
+        verify=True,
+        parallel=True,
+        verbose=True)
+
+
+if __name__ == "__main__":
+    BAZEL_ROOT = utils.find_bazel_root()
+    STYLE_CONF_PATH = BAZEL_ROOT + "/.style.yapf"
+    USER = BAZEL_ROOT.split('/')[2]
+    projects = utils.CHECK_PROJECTS(sys.argv[1:])
+    if "//..." in projects:
+        projects = [p.replace(BAZEL_ROOT, "/")[:-1] for p in glob.glob(BAZEL_ROOT + '/*/')]
+        projects = [p for p in projects if p not in utils.BLACKLISTED_BAZEL_TARGETS]
+
+    for p in projects:
+        if p.endswith("/..."):
+            p = p[:-4]
+        path = BAZEL_ROOT + '/' + p[2:]
+        files = utils.glob_files(path, VALID_PY_FILE_TYPES)
+        if files != []:
+            changed = lint(USER, files, STYLE_CONF_PATH)
+            if changed:
+                print(
+                    "\033[93mWARNING:\033[0m This command modified your files with the recommended linting, you should review the changes before committing"
+                )
+                sys.exit(1)
diff --git a/tools/linter/pylint_diff.py b/tools/linter/pylint_diff.py
@@ -0,0 +1,44 @@
+import os
+import sys
+import glob
+import utils
+import subprocess
+import yapf
+
+VALID_PY_FILE_TYPES = [".py"]
+
+
+def lint(user, target_files, conf, change_file=True):
+    return yapf.FormatFiles(
+        filenames=target_files,
+        lines=None,
+        style_config=conf,
+        no_local_style=None,
+        in_place=change_file,
+        print_diff=False,
+        verify=True,
+        parallel=True,
+        verbose=True)
+
+
+if __name__ == "__main__":
+    BAZEL_ROOT = utils.find_bazel_root()
+    STYLE_CONF_PATH = BAZEL_ROOT + "/.style.yapf"
+    USER = BAZEL_ROOT.split('/')[2]
+    projects = utils.CHECK_PROJECTS(sys.argv[1:])
+    if "//..." in projects:
+        projects = [p.replace(BAZEL_ROOT, "/")[:-1] for p in glob.glob(BAZEL_ROOT + '/*/')]
+        projects = [p for p in projects if p not in utils.BLACKLISTED_BAZEL_TARGETS]
+
+    for p in projects:
+        if p.endswith("/..."):
+            p = p[:-4]
+        path = BAZEL_ROOT + '/' + p[2:]
+        files = utils.glob_files(path, VALID_PY_FILE_TYPES)
+        if files != []:
+            changed = lint(USER, files, STYLE_CONF_PATH)
+            if changed:
+                print(
+                    "\033[93mWARNING:\033[0m This command modified your files with the recommended linting, you should review the changes before committing"
+                )
+                sys.exit(1)
diff --git a/tools/linter/requirements.txt b/tools/linter/requirements.txt
@@ -0,0 +1 @@
+yapf
diff --git a/tools/linter/utils.py b/tools/linter/utils.py
