Optimize Lexer::tokenize()

Author: rvanvelzen

src/Lexer/Lexer.php

@@ -105,16 +105,11 @@ public function tokenize(string $s): array
 		assert($this->regexp !== null);
 		assert($this->types !== null);
 
-		preg_match_all($this->regexp, $s, $tokens, PREG_SET_ORDER);
-
-		$count = count($this->types);
-		foreach ($tokens as &$match) {
-			for ($i = 1; $i <= $count; $i++) {
-				if ($match[$i] !== null && $match[$i] !== '') {
-					$match = [$match[0], $this->types[$i - 1]];
-					break;
-				}
-			}
+		preg_match_all($this->regexp, $s, $matches, PREG_SET_ORDER);
+
+		$tokens = [];
+		foreach ($matches as $match) {
+			$tokens[] = [$match[0], $this->types[count($match) - 2]];
 		}
 
 		$tokens[] = ['', self::TOKEN_END];