Refine newline handling further for the Python3 target

Signed-off-by: Phil Elson <[email protected]>

Author: pelson

runtime/Python3/src/antlr4/xpath/XPathLexer.py

@@ -7,37 +7,27 @@
 
 def serializedATN():
     return [
-        4, 0, 8, 50, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 
-        3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 1, 0, 1, 0, 1, 
-        0, 1, 1, 1, 1, 1, 2, 1, 2, 1, 3, 1, 3, 1, 4, 1, 4, 5, 4, 29, 8, 
-        4, 10, 4, 12, 4, 32, 9, 4, 1, 4, 1, 4, 1, 5, 1, 5, 3, 5, 38, 8, 
-        5, 1, 6, 1, 6, 1, 7, 1, 7, 5, 7, 44, 8, 7, 10, 7, 12, 7, 47, 9, 
-        7, 1, 7, 1, 7, 1, 45, 0, 8, 1, 3, 3, 4, 5, 5, 7, 6, 9, 7, 11, 0, 
-        13, 0, 15, 8, 1, 0, 2, 5, 0, 48, 57, 95, 95, 183, 183, 768, 879, 
-        8255, 8256, 13, 0, 65, 90, 97, 122, 192, 214, 216, 246, 248, 767, 
-        880, 893, 895, 8191, 8204, 8205, 8304, 8591, 11264, 12271, 12289, 
-        55295, 63744, 64975, 65008, 65533, 50, 0, 1, 1, 0, 0, 0, 0, 3, 1, 
-        0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 
-        15, 1, 0, 0, 0, 1, 17, 1, 0, 0, 0, 3, 20, 1, 0, 0, 0, 5, 22, 1, 
-        0, 0, 0, 7, 24, 1, 0, 0, 0, 9, 26, 1, 0, 0, 0, 11, 37, 1, 0, 0, 
-        0, 13, 39, 1, 0, 0, 0, 15, 41, 1, 0, 0, 0, 17, 18, 5, 47, 0, 0, 
-        18, 19, 5, 47, 0, 0, 19, 2, 1, 0, 0, 0, 20, 21, 5, 47, 0, 0, 21, 
-        4, 1, 0, 0, 0, 22, 23, 5, 42, 0, 0, 23, 6, 1, 0, 0, 0, 24, 25, 5, 
-        33, 0, 0, 25, 8, 1, 0, 0, 0, 26, 30, 3, 13, 6, 0, 27, 29, 3, 11, 
-        5, 0, 28, 27, 1, 0, 0, 0, 29, 32, 1, 0, 0, 0, 30, 28, 1, 0, 0, 0, 
-        30, 31, 1, 0, 0, 0, 31, 33, 1, 0, 0, 0, 32, 30, 1, 0, 0, 0, 33, 
-        34, 6, 4, 0, 0, 34, 10, 1, 0, 0, 0, 35, 38, 3, 13, 6, 0, 36, 38, 
-        7, 0, 0, 0, 37, 35, 1, 0, 0, 0, 37, 36, 1, 0, 0, 0, 38, 12, 1, 0, 
-        0, 0, 39, 40, 7, 1, 0, 0, 40, 14, 1, 0, 0, 0, 41, 45, 5, 39, 0, 
-        0, 42, 44, 9, 0, 0, 0, 43, 42, 1, 0, 0, 0, 44, 47, 1, 0, 0, 0, 45, 
-        46, 1, 0, 0, 0, 45, 43, 1, 0, 0, 0, 46, 48, 1, 0, 0, 0, 47, 45, 
-        1, 0, 0, 0, 48, 49, 5, 39, 0, 0, 49, 16, 1, 0, 0, 0, 4, 0, 30, 37, 
-        45, 1, 1, 4, 0
-    ]
+        4,0,8,50,6,-1,2,0,7,0,2,1,7,1,2,2,7,2,2,3,7,3,2,4,7,4,2,5,7,5,2,6,7,6,2,
+        7,7,7,1,0,1,0,1,0,1,1,1,1,1,2,1,2,1,3,1,3,1,4,1,4,5,4,29,8,4,10,4,12,4,
+        32,9,4,1,4,1,4,1,5,1,5,3,5,38,8,5,1,6,1,6,1,7,1,7,5,7,44,8,7,10,7,12,7,
+        47,9,7,1,7,1,7,1,45,0,8,1,3,3,4,5,5,7,6,9,7,11,0,13,0,15,8,1,0,2,5,0,48,
+        57,95,95,183,183,768,879,8255,8256,13,0,65,90,97,122,192,214,216,246,248,
+        767,880,893,895,8191,8204,8205,8304,8591,11264,12271,12289,55295,63744,
+        64975,65008,65533,50,0,1,1,0,0,0,0,3,1,0,0,0,0,5,1,0,0,0,0,7,1,0,0,0,0,
+        9,1,0,0,0,0,15,1,0,0,0,1,17,1,0,0,0,3,20,1,0,0,0,5,22,1,0,0,0,7,24,1,0,
+        0,0,9,26,1,0,0,0,11,37,1,0,0,0,13,39,1,0,0,0,15,41,1,0,0,0,17,18,5,47,0,
+        0,18,19,5,47,0,0,19,2,1,0,0,0,20,21,5,47,0,0,21,4,1,0,0,0,22,23,5,42,0,
+        0,23,6,1,0,0,0,24,25,5,33,0,0,25,8,1,0,0,0,26,30,3,13,6,0,27,29,3,11,5,
+        0,28,27,1,0,0,0,29,32,1,0,0,0,30,28,1,0,0,0,30,31,1,0,0,0,31,33,1,0,0,0,
+        32,30,1,0,0,0,33,34,6,4,0,0,34,10,1,0,0,0,35,38,3,13,6,0,36,38,7,0,0,0,
+        37,35,1,0,0,0,37,36,1,0,0,0,38,12,1,0,0,0,39,40,7,1,0,0,40,14,1,0,0,0,41,
+        45,5,39,0,0,42,44,9,0,0,0,43,42,1,0,0,0,44,47,1,0,0,0,45,46,1,0,0,0,45,
+        43,1,0,0,0,46,48,1,0,0,0,47,45,1,0,0,0,48,49,5,39,0,0,49,16,1,0,0,0,4,0,
+        30,37,45,1,1,4,0
+    ]  # fmt: skip
 
 
 class XPathLexer(Lexer):
-
     atn = ATNDeserializer().deserialize(serializedATN())
 
     decisionsToDFA = [DFA(ds, i) for i, ds in enumerate(atn.decisionToState)]
@@ -95,7 +85,7 @@ class XPathLexer(Lexer):
 
     def __init__(
         self,
-        input = None,
+        input=None,
         output: typing.TextIO = sys.stdout,
     ) -> None:
         super().__init__(input, output)

tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg

@@ -55,6 +55,7 @@ import typing
 
 from antlr4 import *
 
+
 <namedActions.header>
 <parser>
 
@@ -69,16 +70,17 @@ else:
     from <file.parserName> import <file.parserName>
 <header>
 
-# This class defines a complete listener for a parse tree produced by <file.parserName>.
+
 class <file.grammarName>Listener(ParseTreeListener):
+	# This class defines a complete listener for a parse tree produced by <file.parserName>.
 
     <file.listenerNames:{lname |
-# Enter a parse tree produced by <file.parserName>#<lname>.
 def enter<lname; format="cap">(self, ctx: <file.parserName>.<lname; format="cap">Context) -> None:
+	# Enter a parse tree produced by <file.parserName>#<lname>.
     pass
 
-# Exit a parse tree produced by <file.parserName>#<lname>.
 def exit<lname; format="cap">(self, ctx: <file.parserName>.<lname; format="cap">Context) -> None:
+	# Exit a parse tree produced by <file.parserName>#<lname>.
     pass
 
 }; separator="\n">
@@ -97,15 +99,14 @@ else:
     from <file.parserName> import <file.parserName>
 <header>
 
-# This class defines a complete generic visitor for a parse tree produced by <file.parserName>.
 
 class <file.grammarName>Visitor(ParseTreeVisitor):
+    # This class defines a complete generic visitor for a parse tree produced by <file.parserName>.
 
     <file.visitorNames:{lname |
-# Visit a parse tree produced by <file.parserName>#<lname>.
 def visit<lname; format="cap">(self, ctx: <file.parserName>.<lname; format="cap">Context):
+    # Visit a parse tree produced by <file.parserName>#<lname>.
     return self.visitChildren(ctx)
-
 }; separator="\n">
 
 del <file.parserName>
@@ -169,7 +170,6 @@ class <parser.name>(<if(superClass)><superClass><else>Parser<endif>):
 
     <funcs; separator="\n">
 
-
 <if(sempredFuncs)>
     def sempred(self, localctx: RuleContext, ruleIndex: int, predIndex: int):
         if self._predicates is None:
@@ -185,8 +185,6 @@ class <parser.name>(<if(superClass)><superClass><else>Parser<endif>):
     <sempredFuncs.values; separator="\n">
 <endif>
 
-
-
 >>
 
 dumpActions(recog, argFuncs, actionFuncs, sempredFuncs) ::= <<
@@ -252,7 +250,7 @@ def <r.name>_action(self, localctx: <r.ctxType>, actionIndex: int) -> None:
 <elseif(rest(actions))>
     elif actionIndex == <index>:
         <actions.(index)>
-<endif> }; separator="\n\n">
+<endif> }; separator="\n">
 >>
 
 /* This generates a private method since the predIndex is generated, making an
@@ -262,23 +260,19 @@ RuleSempredFunction(r, actions) ::= <<
 def <r.name>_sempred(self, localctx: <r.ctxType>, predIndex: int) -> int:
     <actions:{index|
 <if(first(actions))>
-    if predIndex == <index>:
-        return <actions.(index)>
+if predIndex == <index>:
+    return <actions.(index)>
 <elseif(rest(actions))>
-    elif predIndex == <index>:
-        return <actions.(index)>
-<endif> }; separator="\n">
+elif predIndex == <index>:
+    return <actions.(index)>
+<endif> }; separator="">
 
 >>
 
 RuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,namedActions,finallyAction,postamble,exceptions) ::= <<
-
 <ruleCtx>
-
 <altLabelCtxs:{l | <altLabelCtxs.(l)>}; separator="\n">
-
 def <currentRule.escapedName>(self<currentRule.args:{a | , <a.escapedName><if(a.type)>: <a.type><endif>}>):
-
     localctx = <parser.name>.<currentRule.ctxType>(
         self,
         self._ctx,
@@ -309,10 +303,8 @@ def <currentRule.escapedName>(self<currentRule.args:{a | , <a.escapedName><if(a.
 LeftRecursiveRuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,
 	namedActions,finallyAction,postamble) ::=
 <<
-
 <ruleCtx>
 <altLabelCtxs:{l | <altLabelCtxs.(l)>}; separator="\n">
-
 def <currentRule.escapedName>(self, _p: int = 0<if(currentRule.args)>, <args:{a | , <a>}><endif>) -> <parser.name>.<currentRule.ctxType>:
     _parentctx = self._ctx
     _parentState = self.state
@@ -360,7 +352,6 @@ token = self._input.LA(1)
     pass}; separator="\nel">
 else:
     <error>
-
 >>
 
 LL1OptionalBlock(choice, alts, error) ::= <<
@@ -380,7 +371,6 @@ self._errHandler.sync(self)
 <preamble; separator="\n">
 if <expr>:
     <alts; separator="\n">
-
 <!else if ( !(<followExpr>) ) <error>!>
 >>
 
@@ -394,7 +384,6 @@ while <loopExpr>:
     self.state = <choice.loopBackStateNumber>
     self._errHandler.sync(self)
     <iteration>
-
 >>
 
 LL1PlusBlockSingleAlt(choice, loopExpr, alts, preamble, iteration) ::= <<
@@ -408,7 +397,6 @@ while True:
     <iteration>
     if not (<loopExpr>):
         break
-
 >>
 
 // LL(*) stuff
@@ -424,7 +412,6 @@ if la_ == <i>:
     <alt>
     pass
 }; separator="\nel">
-
 >>
 
 OptionalBlock(choice, alts, error) ::= <<
@@ -435,21 +422,19 @@ la_ = self._interp.adaptivePredict(self._input, <choice.decision>, self._ctx)
 if la_ == <i><if(!choice.ast.greedy)>+1<endif>:
     <alt>
 }; separator="\nel">
-
 >>
 
 StarBlock(choice, alts, sync, iteration) ::= <<
 self.state = <choice.stateNumber>
 self._errHandler.sync(self)
 _alt = self._interp.adaptivePredict(self._input, <choice.decision>, self._ctx)
-while _alt!=<choice.exitAlt> and _alt != ATN.INVALID_ALT_NUMBER:
+while _alt != <choice.exitAlt> and _alt != ATN.INVALID_ALT_NUMBER:
     if _alt == 1<if(!choice.ast.greedy)> + 1<endif>:
         <iteration>
         <alts> <! should only be one !>
     self.state = <choice.loopBackStateNumber>
     self._errHandler.sync(self)
     _alt = self._interp.adaptivePredict(self._input, <choice.decision>, self._ctx)
-
 >>
 
 PlusBlock(choice, alts, error) ::= <<
@@ -466,7 +451,6 @@ if _alt == <i><if(!choice.ast.greedy)> + 1<endif>:
     self.state = <choice.loopBackStateNumber> <! loopback/exit decision !>
     self._errHandler.sync(self)
     _alt = self._interp.adaptivePredict(self._input, <choice.decision>, self._ctx)
-
 >>
 
 Sync(s) ::= "sync(<s.expecting.name>)"
@@ -525,7 +509,7 @@ CommonSetStuff(m, expr, capture, invert) ::= <<
 self.state = <m.stateNumber>
 <if(m.labels)><m.labels:{l | <labelref(l)> = }>self._input.LT(1)<endif>
 <capture>
-<if(invert)>if <m.varName> \<= 0 or <expr><else>if not(<expr>)<endif>:
+<if(invert)>if <m.varName> \<= 0 or <expr><else>if not (<expr>)<endif>:
 <if(m.labels)><m.labels:{l |     <labelref(l)> = }><else>    <endif>self._errHandler.recoverInline(self)
 else:
     self._errHandler.reportMatch(self)
@@ -606,21 +590,23 @@ SetNonLocalAttr(s, rhsChunks)  ::= "self.getInvokingContext(<s.ruleIndex>).<s.es
 
 AddToLabelList(a) ::= "<ctx(a.label)>.<a.listName>.append(<labelref(a.label)>)"
 
-TokenDecl(t) ::= "self.<t.escapedName> = None # <TokenLabelType()>"
-TokenTypeDecl(t) ::= "self.<t.escapedName> = 0 # <TokenLabelType()> type"
-TokenListDecl(t) ::= "self.<t.escapedName> = list() # of <TokenLabelType()>s"
-RuleContextDecl(r) ::= "self.<r.escapedName> = None # <r.ctxName>"
-RuleContextListDecl(rdecl) ::= "self.<rdecl.escapedName> = list() # of <rdecl.ctxName>s"
+TokenDecl(t) ::= "self.<t.escapedName> = None  # <TokenLabelType()>"
+TokenTypeDecl(t) ::= "self.<t.escapedName> = 0  # <TokenLabelType()> type"
+TokenListDecl(t) ::= "self.<t.escapedName> = list()  # of <TokenLabelType()>s"
+RuleContextDecl(r) ::= "self.<r.escapedName> = None  # <r.ctxName>"
+RuleContextListDecl(rdecl) ::= "self.<rdecl.escapedName> = list()  # of <rdecl.ctxName>s"
 
 ContextTokenGetterDecl(t)      ::= <<
 def <t.escapedName>(self):
     return self.getToken(<parser.name>.<t.name>, 0)
+
 >>
 
 // should never be called
 ContextTokenListGetterDecl(t)  ::= <<
 def <t.name>_list(self):
     return self.getTokens(<parser.name>.<t.name>)
+
 >>
 
 ContextTokenListIndexedGetterDecl(t)  ::= <<
@@ -629,6 +615,7 @@ def <t.escapedName>(self, i: typing.Optional[int] = None):
         return self.getTokens(<parser.name>.<t.name>)
     else:
         return self.getToken(<parser.name>.<t.name>, i)
+
 >>
 
 ContextRuleGetterDecl(r)       ::= <<
@@ -683,8 +670,7 @@ class <struct.escapedName>(<if(contextSuperClass)><contextSuperClass><else>Parse
         <attrs:{a | <a>}; separator="\n">
         <struct.ctorAttrs:{a | self.<a.escapedName> = <a.escapedName>}; separator="\n">
 
-    <getters:{g | <g>}; separator="\n\n">
-
+    <getters:{g | <g>}; separator="\n">
     def getRuleIndex(self):
         return <parser.name>.RULE_<struct.derivedFromName>
 
@@ -695,6 +681,7 @@ class <struct.escapedName>(<if(contextSuperClass)><contextSuperClass><else>Parse
 
 <endif>
     <dispatchMethods; separator="\n">
+
     <extensionMembers; separator="\n">
 
 >>
@@ -719,7 +706,7 @@ class <struct.escapedName>(<currentRule.name; format="cap">Context):
 
 ListenerDispatchMethod(method) ::= <<
 def <if(method.isEnter)>enter<else>exit<endif>Rule(self, listener: ParseTreeListener):
-    if hasattr( listener, "<if(method.isEnter)>enter<else>exit<endif><struct.derivedFromName; format="cap">" ):
+    if hasattr(listener, "<if(method.isEnter)>enter<else>exit<endif><struct.derivedFromName; format="cap">"):
         listener.<if(method.isEnter)>enter<else>exit<endif><struct.derivedFromName; format="cap">(self)
 
 >>
@@ -730,7 +717,6 @@ def accept(self, visitor: ParseTreeVisitor):
         return visitor.visit<struct.derivedFromName; format="cap">(self)
     else:
         return visitor.visitChildren(self)
-
 >>
 
 AttributeDecl(d) ::= "self.<d.escapedName> = <if(d.initValue)><d.initValue><else>None<endif>"
@@ -807,7 +793,6 @@ else:
 
 
 class <lexer.name>(<if(superClass)><superClass><else>Lexer<endif>):
-
     atn = ATNDeserializer().deserialize(serializedATN())
 
     decisionsToDFA = [DFA(ds, i) for i, ds in enumerate(atn.decisionToState)]
@@ -852,7 +837,7 @@ class <lexer.name>(<if(superClass)><superClass><else>Lexer<endif>):
 
     def __init__(
         self,
-        input = None,
+        input=None,
         output: typing.TextIO = sys.stdout,
     ) -> None:
         super().__init__(input, output)
@@ -875,8 +860,8 @@ class <lexer.name>(<if(superClass)><superClass><else>Lexer<endif>):
 SerializedATN(model) ::= <<
 def serializedATN():
     return [
-        <model.serialized: {s | <s>}; separator=", ", wrap>
-    ]
+        <model.serialized: {s | <s>}; separator=",", wrap>
+    ]  # fmt: skip
 >>
 
 /** Using a type to init value map, try to init a type; if not in table

tool/src/org/antlr/v4/codegen/target/Python3Target.java

@@ -62,6 +62,8 @@ public class Python3Target extends Target {
 
 	public Python3Target(CodeGenerator gen) {
 		super(gen);
+		// note that lineWidth isn't a hard upper limit.
+		gen.lineWidth = 79;
 	}
 
 	@Override