Fix //$ editor comment parsing for DECORATE and ZScript

Hopefully the ZScript part is ok Fix #1774
sirjuddington · Jan 30, 2025 · c71f584 · c71f584
1 parent d6ffbe3
commit c71f584
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 23 deletions.
diff --git a/src/Game/Decorate.cpp b/src/Game/Decorate.cpp
@@ -283,8 +283,7 @@ void parseDecorateActor(Tokenizer& tz, std::map<int, ThingType>& types, vector<T
 			if (strutil::startsWith(token, "//$"))
 			{
 				// Doom Builder magic editor comment
-				editor_properties.emplace_back(Tokenizer::parseEditorComment(token));
-				tz.advToNextLine();
+				editor_properties.emplace_back(Tokenizer::parseEditorComment(tz.getLine(true)));
 				continue;
 			}
 
@@ -568,6 +567,7 @@ void parseDecorateEntry(ArchiveEntry* entry, std::map<int, ThingType>& types, ve
 	// Init tokenizer
 	Tokenizer tz;
 	tz.setSpecialCharacters(":,{}");
+	tz.enableEditorComments();
 	tz.openMem(entry->data(), entry->name());
 
 	// --- Parse ---

diff --git a/src/Game/ZScript.cpp b/src/Game/ZScript.cpp
@@ -208,6 +208,7 @@ void parseBlocks(ArchiveEntry* entry, vector<ParsedStatement>& parsed, vector<Ar
 	Tokenizer tz;
 	tz.setSpecialCharacters(Tokenizer::DEFAULT_SPECIAL_CHARACTERS + "()+-[]&!?.<>");
 	tz.setCommentTypes(Tokenizer::CommentTypes::CPPStyle | Tokenizer::CommentTypes::CStyle);
+	tz.enableEditorComments();
 	tz.openMem(entry->data(), "ZScript");
 
 	entry_stack.push_back(entry);
@@ -869,9 +870,16 @@ bool Class::parseClassBlock(vector<ParsedStatement>& block)
 		else if (strutil::equalCI(first_token, "states"))
 			states_.parse(statement);
 
-		// DB property comment
+		// DB property comment(s)
 		else if (strutil::startsWith(first_token, editor_comment_prefix))
+		{
 			db_properties_.emplace_back(Tokenizer::parseEditorComment(first_token));
+			for (auto i = 1; i < statement.tokens.size(); ++i)
+			{
+				if (strutil::startsWith(statement.tokens[i], editor_comment_prefix))
+					db_properties_.emplace_back(Tokenizer::parseEditorComment(statement.tokens[i]));
+			}
+		}
 
 		// Function
 		else if (Function::isFunction(statement))

diff --git a/src/Utility/Tokenizer.cpp b/src/Utility/Tokenizer.cpp
@@ -82,7 +82,7 @@ bool isWhitespace(char p)
 std::pair<string, string> Tokenizer::parseEditorComment(string_view token)
 {
 	// Find the first space
-	size_t spos = token.find(" \t");
+	size_t spos = token.find_first_of(" \t");
 	if (spos == string_view::npos)
 	{
 		string key(token.substr(3));
@@ -721,7 +721,7 @@ void Tokenizer::reset()
 // Checks if a comment begins at the current position and returns the comment
 // type if one does (0 otherwise)
 // -----------------------------------------------------------------------------
-unsigned Tokenizer::checkCommentBegin()
+unsigned Tokenizer::checkCommentBegin() const
 {
 	// C-Style comment (/*)
 	if (comment_types_ & CStyle && state_.position + 1 < state_.size && data_[state_.position] == '/'
@@ -838,13 +838,12 @@ void Tokenizer::tokenizeToken()
 	}
 
 	// Check for end of token
-	if (isWhitespace(data_[state_.position]) ||       // Whitespace
-		isSpecialCharacter(data_[state_.position]) || // Special character
-		checkCommentBegin() > 0)                      // Comment
+	if (isEndOfToken())
 	{
 		// End token
-		state_.state = TokenizeState::State::Unknown;
-		state_.done  = true;
+		state_.state  = TokenizeState::State::Unknown;
+		state_.done   = true;
+		state_.to_eol = false;
 
 		return;
 	}
@@ -859,7 +858,7 @@ void Tokenizer::tokenizeToken()
 void Tokenizer::tokenizeComment()
 {
 	// Check for decorate //$
-	if (decorate_ && state_.comment_type == CPPStyle)
+	if (editor_comments_ && state_.comment_type == CPPStyle)
 	{
 		if (data_[state_.position] == '$' && data_[state_.position - 1] == '/' && data_[state_.position - 2] == '/')
 		{
@@ -868,6 +867,7 @@ void Tokenizer::tokenizeComment()
 			state_.current_token.quoted_string = false;
 			state_.current_token.pos_start     = state_.position - 2;
 			state_.state                       = TokenizeState::State::Token;
+			state_.to_eol                      = true;
 			return;
 		}
 	}
@@ -991,6 +991,18 @@ void Tokenizer::resetToLineStart()
 	}
 }
 
+bool Tokenizer::isEndOfToken() const
+{
+	// Token is to end of line, only ends on newline
+	if (state_.to_eol)
+		return data_[state_.position] == '\n';
+
+	// Regular token
+	return isWhitespace(data_[state_.position]) ||       // Whitespace
+		   isSpecialCharacter(data_[state_.position]) || // Special character
+		   checkCommentBegin() > 0;                      // Comment
+}
+
 
 // Testing
 

diff --git a/src/Utility/Tokenizer.h b/src/Utility/Tokenizer.h
@@ -67,7 +67,8 @@ class Tokenizer
 		unsigned current_line = 1;
 		unsigned comment_type = 0;
 		Token    current_token;
-		bool     done = false;
+		bool     done   = false;
+		bool     to_eol = false;
 	};
 
 	static std::pair<string, string> parseEditorComment(string_view token);
@@ -77,7 +78,7 @@ class Tokenizer
 
 	// Accessors
 	const string& source() const { return source_; }
-	bool          decorate() const { return decorate_; }
+	bool          decorate() const { return editor_comments_; }
 	bool          readLowerCase() const { return read_lowercase_; }
 	const Token&  current() const { return token_current_; }
 	const Token&  peek() const;
@@ -90,8 +91,8 @@ class Tokenizer
 	}
 	void setSource(const wxString& source) { source_ = source; }
 	void setReadLowerCase(bool lower) { read_lowercase_ = lower; }
-	void enableDecorate(bool enable) { decorate_ = enable; }
-	void enableDebug(bool enable) { debug_ = enable; }
+	void enableEditorComments(bool enable = true) { editor_comments_ = enable; }
+	void enableDebug(bool enable = true) { debug_ = enable; }
 
 	// Token Iterating
 	const Token&  next();
@@ -212,25 +213,26 @@ class Tokenizer
 	TokenizeState state_         = {};
 
 	// Configuration
-	int          comment_types_;          // Types of comments to skip
-	vector<char> special_characters_;     // These will always be read as separate tokens
-	string       source_;                 // What file/entry/chunk is being tokenized
-	bool         decorate_       = false; // Special handling for //$ comments
-	bool         read_lowercase_ = false; // If true, tokens will all be read in lowercase
-										  // (except for quoted strings, obviously)
-	bool debug_ = false;                  // Log each token read
+	int          comment_types_;           // Types of comments to skip
+	vector<char> special_characters_;      // These will always be read as separate tokens
+	string       source_;                  // What file/entry/chunk is being tokenized
+	bool         editor_comments_ = false; // Special handling for //$ comments
+	bool         read_lowercase_  = false; // If true, tokens will all be read in lowercase
+										   // (except for quoted strings, obviously)
+	bool debug_ = false;                   // Log each token read
 
 	// Static
 	static Token invalid_token_;
 
 	// Tokenizing
-	unsigned checkCommentBegin();
+	unsigned checkCommentBegin() const;
 	void     tokenizeUnknown();
 	void     tokenizeToken();
 	void     tokenizeComment();
 	void     tokenizeWhitespace();
 	bool     readNext(Token* target);
 	bool     readNext() { return readNext(&token_next_); }
 	void     resetToLineStart();
+	bool     isEndOfToken() const;
 };
 } // namespace slade