From e193f17cfb4ada4b0172f0b7b0c7544f032ed577 Mon Sep 17 00:00:00 2001
From: Theodore Tsirpanis <teo@tsirpanis.gr>
Date: Tue, 29 Aug 2023 02:39:17 +0300
Subject: [PATCH] Track in ParserState if suspending the tokenizer is
 supported.

I thought that suspending the tokenizer without being in a chain would have no effect, but it actually has the effect of prohibiting subsequent suspensions.
With this change, suspending is a real no-op if the tokenizer is not wrapped in a chain.
---
 src/FarkleNeo/Parser/ParserState.cs                   | 11 +++++++++++
 src/FarkleNeo/Parser/Tokenizers/ChainedTokenizer.cs   |  4 ++++
 .../Parser/Tokenizers/TokenizerExtensions.cs          | 10 ++++++++++
 3 files changed, 25 insertions(+)

diff --git a/src/FarkleNeo/Parser/ParserState.cs b/src/FarkleNeo/Parser/ParserState.cs
index 300c4ab2..c106f356 100644
--- a/src/FarkleNeo/Parser/ParserState.cs
+++ b/src/FarkleNeo/Parser/ParserState.cs
@@ -63,6 +63,17 @@ public struct ParserState
     /// <seealso cref="ParserExtensions.ParseFileAsync"/>
     public string? InputName { get; set; }
 
+    /// <summary>
+    /// Whether suspending the tokenizer will have any effect.
+    /// </summary>
+    /// <remarks>
+    /// We must have this property because if the tokenizer is not wrapped in a chain
+    /// and suspends to itself (which is still supported), there is no chain to "unsuspend"
+    /// the tokenizer, and when the tokenizer returns and wants to suspend again, it will
+    /// throw with a message that it is already suspended.
+    /// </remarks>
+    internal bool TokenizerSupportsSuspending { get; set; }
+
     internal void Consume<T>(ReadOnlySpan<T> characters)
     {
         _positionTracker.Advance(characters);
diff --git a/src/FarkleNeo/Parser/Tokenizers/ChainedTokenizer.cs b/src/FarkleNeo/Parser/Tokenizers/ChainedTokenizer.cs
index 24c70c46..df517ff6 100644
--- a/src/FarkleNeo/Parser/Tokenizers/ChainedTokenizer.cs
+++ b/src/FarkleNeo/Parser/Tokenizers/ChainedTokenizer.cs
@@ -39,6 +39,10 @@ internal static Tokenizer<TChar> Create(ImmutableArray<Tokenizer<TChar>> compone
 
     public override bool TryGetNextToken(ref ParserInputReader<TChar> input, ITokenSemanticProvider<TChar> semanticProvider, out TokenizerResult result)
     {
+        // We mark this parser operation as supporting suspending.
+        // It might cause some issues if the tokenizer changes in the middle of the
+        // operation but this is not a supported scenario.
+        input.State.TokenizerSupportsSuspending = true;
         // Get the state of the chained tokenizer. If we have not suspended before,
         // it will be null.
         var tokenizerState = input.GetChainedTokenizerStateOrNull();
diff --git a/src/FarkleNeo/Parser/Tokenizers/TokenizerExtensions.cs b/src/FarkleNeo/Parser/Tokenizers/TokenizerExtensions.cs
index 322e2c47..d6bcdb94 100644
--- a/src/FarkleNeo/Parser/Tokenizers/TokenizerExtensions.cs
+++ b/src/FarkleNeo/Parser/Tokenizers/TokenizerExtensions.cs
@@ -78,6 +78,11 @@ private static void SuspendTokenizerCore<TChar>(this ref ParserInputReader<TChar
     public static void SuspendTokenizer<TChar>(this ref ParserInputReader<TChar> input, Tokenizer<TChar> tokenizer)
     {
         ArgumentNullExceptionCompat.ThrowIfNull(tokenizer);
+
+        if (!input.State.TokenizerSupportsSuspending)
+        {
+            return;
+        }
         input.SuspendTokenizerCore(tokenizer);
     }
 
@@ -119,6 +124,11 @@ public static void SuspendTokenizer<TChar, TArg>(this ref ParserInputReader<TCha
         ITokenizerResumptionPoint<TChar, TArg> resumptionPoint, TArg arg)
     {
         ArgumentNullExceptionCompat.ThrowIfNull(resumptionPoint);
+
+        if (!input.State.TokenizerSupportsSuspending)
+        {
+            return;
+        }
         input.SuspendTokenizerCore(new TokenizerResumptionPoint<TChar, TArg>(resumptionPoint, arg));
     }