PoC for the event-based plugin system with YARD parsing plugin

lib/rdoc.rb

@@ -208,4 +208,6 @@ def self.home
   autoload :Extend,         "#{__dir__}/rdoc/code_object/extend"
   autoload :Require,        "#{__dir__}/rdoc/code_object/require"
 
+  autoload :BasePlugin,     "#{__dir__}/rdoc/base_plugin"
+  autoload :EventRegistry,  "#{__dir__}/rdoc/event_registry"
 end

lib/rdoc/base_plugin.rb

@@ -0,0 +1,20 @@
+module RDoc
+  class BasePlugin
+    # Register a literner for the given event
+
+    def self.listens_to(event_name, &block)
+      rdoc.event_registry.register(event_name, block)
+    end
+
+    # Activate the plugin with the given RDoc instance
+    # Without calling this, plugins won't work
+
+    def self.activate_with(rdoc = ::RDoc::RDoc.current)
+      @@rdoc = rdoc
+    end
+
+    def self.rdoc
+      @@rdoc
+    end
+  end
+end

lib/rdoc/event_registry.rb

 end 
 ## 
 # Adds a post-process handler for directives.  The handler will be called 
 # with the result RDoc::Comment (or text String) and the code object for the 
 # comment (if any). 
 def self.post_process &block 
   @post_processors << block 
 end 
@@ -0,0 +1,26 @@
+module RDoc
+  class EventRegistry
+    EVENT_TYPES = %i[
+      rdoc_start
+      sample
+      rdoc_store_complete
+    ]
+
+    attr_reader :environment
+
+    def initialize
+      @registry = EVENT_TYPES.map { |event_name| [event_name, []] }.to_h
+      @environment = {}
+    end
+
+    def register(event_name, handler)
+      @registry[event_name] << handler
+    end
+
+    def trigger(event_name, *args)
+      @registry[event_name].each do |handler|
+        handler.call(@environment, *args)
+      end
+    end
+  end
+end

lib/rdoc/options.rb

@@ -268,6 +268,11 @@ class RDoc::Options
 
   attr_accessor :pipe
 
+  ##
+  # Currently enabled plugins
+
+  attr_reader :plugins
+
   ##
   # Array of directories to search for files to satisfy an :include:
 
@@ -395,6 +400,7 @@ def init_ivars # :nodoc:
     @coverage_report = false
     @op_dir = nil
     @page_dir = nil
+    @plugins = []
     @pipe = false
     @output_decoration = true
     @rdoc_include = []
@@ -436,6 +442,7 @@ def init_with map # :nodoc:
     @main_page      = map['main_page']
     @markup         = map['markup']
     @op_dir         = map['op_dir']
+    @plugins        = map['plugins']
     @show_hash      = map['show_hash']
     @tab_width      = map['tab_width']
     @template_dir   = map['template_dir']
@@ -503,6 +510,7 @@ def == other # :nodoc:
       @main_page      == other.main_page      and
       @markup         == other.markup         and
       @op_dir         == other.op_dir         and
+      @plugins        == other.plugins        and
       @rdoc_include   == other.rdoc_include   and
       @show_hash      == other.show_hash      and
       @static_path    == other.static_path    and
@@ -868,6 +876,12 @@ def parse argv
 
       opt.separator nil
 
+      opt.on("--plugins=PLUGINS", "-P", Array, "Use plugins") do |value|
+        @plugins.concat value
+      end
+
+      opt.separator nil
+
       opt.on("--tab-width=WIDTH", "-w", Integer,
              "Set the width of tab characters.") do |value|
         raise OptionParser::InvalidArgument,
@@ -1344,6 +1358,16 @@ def visibility= visibility
     end
   end
 
+  # Load plugins specified with options
+  # Currently plugin search logic is very simple, but it's not practical.
+  # TODO: We will improve this later.
+
+  def load_plugins
+    @plugins.each do |plugin_name|
+      require_relative "./#{plugin_name}.rb"
+    end
+  end
+
   ##
   # Displays a warning using Kernel#warn if we're being verbose
 

lib/rdoc/rdoc.rb

@@ -71,6 +71,11 @@ class RDoc::RDoc
 
   attr_accessor :store
 
+  ##
+  # Event registry for RDoc plugins
+
+  attr_accessor :event_registry
+
   ##
   # Add +klass+ that can generate output after parsing
 
@@ -105,6 +110,7 @@ def initialize
     @options       = nil
     @stats         = nil
     @store         = nil
+    @event_registry = ::RDoc::EventRegistry.new
   end
 
   ##
@@ -449,6 +455,9 @@ def document options
     end
     @options.finish
 
+    ::RDoc::BasePlugin.activate_with(self)
+    @options.load_plugins
+
     @store = RDoc::Store.new(@options)
 
     if @options.pipe then
@@ -469,6 +478,7 @@ def document options
     @options.default_title = "RDoc Documentation"
 
     @store.complete @options.visibility
+    @event_registry.trigger :rdoc_store_complete, @store
 
     @stats.coverage_level = @options.coverage_report
 

lib/rdoc/yard_plugin.rb

@@ -0,0 +1,228 @@
+# Yard type parser is inspired by the following code:
+# https://github.com/lsegal/yard-types-parser/blob/master/lib/yard_types_parser.rb
+
+require_relative 'base_plugin'
+require 'strscan'
+
+module RDoc
+  class YardPlugin < BasePlugin
+    listens_to :rdoc_store_complete do |env, store|
+      store.all_classes_and_modules.each do |cm|
+        cm.each_method do |meth|
+          puts "Parsing #{meth.name}"
+          parsed_comment = Parser.new(meth.comment.text).parse
+          # meth.params = parsed_comment.param.map(&:to_s).join("\n")
+          meth.comment.text = parsed_comment.plain.join("\n")
+        end
+      end
+    end
+
+    class Parser
+      ParamData = Struct.new(:type, :name, :desc, keyword_init: true) do
+        def append_desc(line)
+          self[:desc] += line
+        end
+
+        def to_s
+          "Name: #{self[:name]}, Type: #{self[:type].map(&:to_s).join(' or ')}, Desc: #{self[:desc]}"
+        end
+      end
+      ReturnData = Struct.new(:type, :desc, keyword_init: true)
+      RaiseData = Struct.new(:type, :desc, keyword_init: true)
+      ParsedComment = Struct.new(:param, :return, :raise, :plain)
+
+      TAG_PARSING_REGEXES = {
+        param: /
+        @param\s+
+          (?:                                 # Match either of the following:
+          \[(?<type1>[^\]]+)\]\s+(?<name1>\S+)\s*(?<desc1>.*)? |  # [Type] name desc
+          (?<name2>\S+)\s+\[(?<type2>[^\]]+)\]\s*(?<desc2>.*)?   # name [Type] desc
+          )
+        /x,
+          return: /@return\s+\[(?<type>[^\]]+)\]\s*(?<desc>.*)?/,
+          raise: /@raise\s+\[(?<type>[^\]]+)\]\s*(?<desc>.*)?/
+      }
+      def initialize(comment)
+        @comment = comment
+        @parsed_comment = ParsedComment.new([], nil, [], [])
+        @mode = :initial
+        @base_indentation_level = 0 # @comment.lines.first[/^#\s*/].size
+      end
+
+      def parse
+        @comment.each_line do |line|
+          current_indentation_level = line[/^#\s*/]&.size || 0
+          if current_indentation_level >= @base_indentation_level + 2
+            # Append to the previous tag
+            data = @mode == :param ? @parsed_comment[@mode].last : @parsed_comment[@mode]
+            data.append_desc(line)
+          else
+            if (tag, matchdata = matching_any_tag(line))
+              if tag == :param
+                type = matchdata[:type1] || matchdata[:type2]
+                name = matchdata[:name1] || matchdata[:name2]
+                desc = matchdata[:desc1] || matchdata[:desc2]
+                parsed_type = TypeParser.parse(type)
+                @parsed_comment[:param] << ParamData.new(type: parsed_type, name: name, desc: desc)
+                @mode = :param
+              elsif tag == :return
+                type = matchdata[:type]
+                desc = matchdata[:desc]
+                parsed_type = TypeParser.parse(type)
+                @parsed_comment[:return] = ReturnData.new(type: parsed_type, desc: desc)
+                @mode = :return
+              elsif tag == :raise
+                type = matchdata[:type]
+                desc = matchdata[:desc]
+                parsed_type = TypeParser.parse(type)
+                @parsed_comment[:raise] << RaiseData.new(type: parsed_type, desc: desc)
+                @mode = :raise
+              end
+            else
+              @parsed_comment[:plain] << line
+            end
+          end
+          @base_indentation_level = current_indentation_level
+        end
+
+        @parsed_comment
+      end
+
+      private
+
+      def matching_any_tag(line)
+        TAG_PARSING_REGEXES.each do |tag, regex|
+          matchdata = line.match(regex)
+          return [tag, matchdata] if matchdata
+        end
+        nil
+      end
+    end
+
+    class Type
+      attr_reader :name
+
+      def initialize(name)
+        @name = name
+      end
+
+      def to_s
+        @name
+      end
+    end
+
+    class CollectionType < Type
+      attr_reader :type
+
+      def initialize(name, type)
+        super(name)
+        @type = type
+      end
+
+      def to_s
+        "#{@name}<#{@type}>"
+      end
+    end
+
+    class FixedCollectionType < Type
+      attr_reader :type
+
+      def initialize(name, type)
+        super(name)
+        @type = type
+      end
+
+      def to_s
+        "#{@name}(#{@type})"
+      end
+    end
+
+    class HashCollectionType < Type
+      attr_reader :key_type, :value_type
+
+      def initialize(name, key_type, value_type)
+        super(name)
+        @key_type = key_type
+        @value_type = value_type
+      end
+
+      def to_s
+        "#{@name}<#{@key_type} => #{@value_type}>"
+      end
+    end
+
+    class TypeParser
+      TOKENS = {
+        collection_start: /</,
+        collection_end: />/,
+        fixed_collection_start: /\(/,
+        fixed_collection_end: /\)/,
+        type_name: /#\w+|((::)?\w+)+/,
+        literal: /(?:
+           '(?:\\'|[^'])*' |
+           "(?:\\"|[^"])*" |
+           :[a-zA-Z_][a-zA-Z0-9_]*|
+           \b(?:true|false|nil)\b |
+           \b\d+(?:\.\d+)?\b
+          )/x,
+        type_next: /[,;]/,
+        whitespace: /\s+/,
+        hash_collection_start: /\{/,
+        hash_collection_next: /=>/,
+        hash_collection_end: /\}/,
+        parse_end: nil
+      }
+
+      def self.parse(string)
+        new(string).parse
+      end
+
+      def initialize(string)
+        @scanner = StringScanner.new(string)
+      end
+
+      def parse
+        types = []
+        type = nil
+        fixed = false
+        name = nil
+        loop do
+          found = false
+          TOKENS.each do |token_type, match|
+            if (match.nil? && @scanner.eos?) || (match && token = @scanner.scan(match))
+              found = true
+              case token_type
+              when :type_name, :literal
+                raise SyntaxError, "expecting END, got name '#{token}'" if name
+                name = token
+              when :type_next
+                raise SyntaxError, "expecting name, got '#{token}' at #{@scanner.pos}" if name.nil?
+                unless type
+                  type = Type.new(name)
+                end
+                types << type
+                type = nil
+                name = nil
+              when :fixed_collection_start, :collection_start
+                name ||= "Array"
+                klass = token_type == :collection_start ? CollectionType : FixedCollectionType
+                type = klass.new(name, parse)
+              when :hash_collection_start
+                name ||= "Hash"
+                type = HashCollectionType.new(name, parse, parse)
+              when :hash_collection_next, :hash_collection_end, :fixed_collection_end, :collection_end, :parse_end
+                raise SyntaxError, "expecting name, got '#{token}'" if name.nil?
+                unless type
+                  type = Type.new(name)
+                end
+                types << type
+                return types
+              end
+            end
+          end
+          raise SyntaxError, "invalid character at #{@scanner.peek(1)}" unless found
+        end
+      end
+    end
+  end
+end

test/rdoc/test_rdoc_options.rb

@@ -75,6 +75,7 @@ def test_to_yaml
       'markup'                => 'rdoc',
       'output_decoration'     => true,
       'page_dir'              => nil,
+      'plugins'               => [],
       'rdoc_include'          => [],
       'show_hash'             => false,
       'static_path'           => [],