Add language-specific node filtering

Gemfile

@@ -2,7 +2,8 @@
 source "https://rubygems.org"
 
 gem "concurrent-ruby", "~> 1.0.0"
-gem "flay"
+gem "flay", "~> 2.9"
+gem "sexp_processor", "~> 4.10including-betas0" # TODO: Remove when flay >= 2.10
 gem "json"
 
 group :test do

Gemfile.lock

@@ -5,7 +5,7 @@ GEM
     concurrent-ruby (1.0.0)
     diff-lcs (1.2.5)
     erubis (2.7.0)
-    flay (2.8.1)
+    flay (2.9.0)
       erubis (~> 2.7.0)
       path_expander (~> 1.0)
       ruby_parser (~> 3.0)
@@ -31,21 +31,19 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.3.0)
     rspec-support (3.3.0)
-    ruby_parser (3.8.4)
+    ruby_parser (3.9.0)
       sexp_processor (~> 4.1)
-    sexp_processor (4.8.0)
+    sexp_processor (4.10.0b1)
     slop (3.6.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   concurrent-ruby (~> 1.0.0)
-  flay
+  flay (~> 2.9)
   json
   pry
   rake
   rspec
-
-BUNDLED WITH
-   1.14.4
+  sexp_processor (~> 4.10including.pre.betas0)

README.md

@@ -126,6 +126,33 @@ engines:
             - "**/*.ruby"
 ```
 
+### Node Filtering
+
+Sometimes structural similarities are reported that you just don't
+care about. For example, the contents of arrays or hashes might have
+similar structures and there's little you can do to refactor them. You
+can specify language specific filters to ignore any issues that match
+the pattern. Here is an example that filters simple hashes and arrays:
+
+```yaml
+engines:
+  duplication:
+    enabled: true
+    config:
+      languages:
+        ruby:
+          filters:
+            - "(hash (lit _) (str _) ___)"
+            - "(array (str _) ___)"
+```
+
+The syntax for patterns are pretty simple. In the first pattern:
+`"(hash (lit _) (str _) ___)"` specifies "A hash with a literal key, a
+string value, followed by anything else (including nothing)". You
+could also specify `"(hash ___)"` to ignore all hashes altogether.
+
+For more information on pattern matching,
+see [sexp_processor][sexp_processor], especially [sexp.rb][sexp.rb]
 
 
 [codeclimate]: https://codeclimate.com/dashboard
@@ -134,3 +161,5 @@ engines:
 [cli]: https://github.com/codeclimate/codeclimate
 [rule-of-three]: https://en.wikipedia.org/wiki/Rule_of_three_(computer_programming)
 [exclude-files-engine]: https://docs.codeclimate.com/docs/excluding-files-and-folders#section-exclude-paths-for-specific-engines
+[sexp_processor]: https://github.com/seattlerb/sexp_processor/
+[sexp.rb]: https://github.com/seattlerb/sexp_processor/blob/master/lib/sexp.rb

lib/cc/engine/analyzers/analyzer_base.rb

@@ -40,12 +40,20 @@ def files
           file_list.files
         end
 
+        def filters
+          engine_config.filters_for(language)
+        end
+
+        def language
+          self.class::LANGUAGE
+        end
+
         def mass_threshold
-          engine_config.mass_threshold_for(self.class::LANGUAGE) || self.class::DEFAULT_MASS_THRESHOLD
+          engine_config.mass_threshold_for(language) || self.class::DEFAULT_MASS_THRESHOLD
         end
 
         def count_threshold
-          engine_config.count_threshold_for(self.class::LANGUAGE)
+          engine_config.count_threshold_for(language)
         end
 
         def calculate_points(mass)
@@ -77,7 +85,7 @@ def file_list
           @_file_list ||= ::CC::Engine::Analyzers::FileList.new(
             engine_config: engine_config,
             patterns: engine_config.patterns_for(
-              self.class::LANGUAGE,
+              language,
               self.class::PATTERNS,
             ),
           )

lib/cc/engine/analyzers/engine_config.rb

@@ -27,6 +27,12 @@ def concurrency
           config.fetch("config", {}).fetch("concurrency", 2).to_i
         end
 
+        def filters_for(language)
+          fetch_language(language).fetch("filters", []).map { |filter|
+            Sexp::Matcher.parse filter
+          }
+        end
+
         def mass_threshold_for(language)
           threshold = fetch_language(language).fetch("mass_threshold", nil)
 

lib/cc/engine/analyzers/javascript/main.rb

@@ -28,7 +28,8 @@ def transform_sexp(sexp)
           private
 
           def process_file(path)
-            Node.new(js_parser.new(File.read(path), path).parse.syntax_tree, path).format
+            ast = js_parser.new(File.read(path), path).parse
+            Node.new(ast.syntax_tree, path).format if ast
           end
 
           def js_parser

lib/cc/engine/analyzers/javascript/parser.rb

@@ -7,6 +7,8 @@ module Engine
     module Analyzers
       module Javascript
         class Parser < ParserBase
+          TIMEOUT = 10
+
           attr_reader :code, :filename, :syntax_tree
 
           def initialize(code, filename)
@@ -15,12 +17,14 @@ def initialize(code, filename)
           end
 
           def parse
-            runner = CommandLineRunner.new(js_command)
+            runner = CommandLineRunner.new(js_command, TIMEOUT)
             runner.run(strip_shebang(code)) do |ast|
               @syntax_tree = parse_json(ast)
             end
 
             self
+          rescue Timeout::Error
+            warn "TIMEOUT parsing #{filename}. Skipping."
           end
 
           private

lib/cc/engine/analyzers/php/main.rb

@@ -27,10 +27,8 @@ def transform_sexp(sexp)
 
           def process_file(path)
             code = File.binread(path)
-            parser = php_parser.new(code, path).parse
-            syntax_tree = parser.syntax_tree
-
-            syntax_tree&.to_sexp
+            ast = php_parser.new(code, path).parse
+            ast.syntax_tree&.to_sexp if ast
           end
 
           def php_parser

lib/cc/engine/analyzers/php/parser.rb

@@ -10,6 +10,8 @@ module Engine
     module Analyzers
       module Php
         class Parser < ParserBase
+          TIMEOUT = 10
+
           attr_reader :code, :filename, :syntax_tree
 
           def initialize(code, filename)
@@ -18,7 +20,8 @@ def initialize(code, filename)
           end
 
           def parse
-            runner = CommandLineRunner.new("php -d 'display_errors = Off' #{parser_path}")
+            cmd = "php -d 'display_errors = Off' #{parser_path}"
+            runner = CommandLineRunner.new(cmd, TIMEOUT)
             runner.run(code) do |output|
               json = parse_json(output)
 
@@ -29,6 +32,8 @@ def parse
             end
 
             self
+          rescue Timeout::Error
+            warn "TIMEOUT parsing #{filename}. Skipping."
           end
 
           private

lib/cc/engine/analyzers/python/parser.rb

@@ -9,6 +9,7 @@ module Engine
     module Analyzers
       module Python
         class Parser < ParserBase
+          TIMEOUT = 10
           attr_reader :code, :filename, :syntax_tree
 
           def initialize(python_version, code, filename)
@@ -18,12 +19,14 @@ def initialize(python_version, code, filename)
           end
 
           def parse
-            runner = CommandLineRunner.new(python_command)
+            runner = CommandLineRunner.new(python_command, TIMEOUT)
             runner.run(code) do |ast|
               @syntax_tree = parse_json(ast)
             end
 
             self
+          rescue Timeout::Error
+            warn "TIMEOUT parsing #{filename}. Skipping."
           end
 
           private

lib/cc/engine/analyzers/reporter.rb

@@ -10,8 +10,6 @@ module CC
   module Engine
     module Analyzers
       class Reporter
-        TIMEOUT = 10
-
         def initialize(engine_config, language_strategy, io)
           @engine_config = engine_config
           @language_strategy = language_strategy
@@ -87,17 +85,12 @@ def new_violations(issue)
         end
 
         def flay_options
-          {
-            diff: false,
+          changes = {
             mass: language_strategy.mass_threshold,
-            summary: false,
-            verbose: false,
-            number: true,
-            timeout: TIMEOUT,
-            liberal: false,
-            fuzzy: false,
-            only: nil,
+            filters: language_strategy.filters,
           }
+
+          CCFlay.default_options.merge changes
         end
 
         def debug(message)

lib/cc/engine/analyzers/ruby/main.rb

@@ -20,7 +20,7 @@ class Main < CC::Engine::Analyzers::Base
           ].freeze
           DEFAULT_MASS_THRESHOLD = 18
           POINTS_PER_OVERAGE = 100_000
-          TIMEOUT = 300
+          TIMEOUT = 30
 
           private