feat: reframed comparison as project-vs-service

Author: JoshuaKGoldberg

README.md

@@ -20,8 +20,8 @@ See [sharkdp/hyperfine#installation](https://github.com/sharkdp/hyperfine#instal
 
 ```shell
 npm install
-npm generate
-npm measure
+npm run generate
+npm run measure
 ```
 
 ### Measured Attributes
@@ -38,20 +38,19 @@ The `caseEntries` values in `src/data.ts` can be modified to test:
 
 ## Results
 
-Right now, `parserOptions.project` outperforms `parserOptions.projectService`.
+Right now, `parserOptions.project` _with_ single-run inference outperforms `parserOptions.projectService`.
 This is a performance issue and we are investigating it as a critical bug for v8.
 
 ```plaintext
-┌───────┬──────────────────────┬──────────────────────┬──────────────────────┬──────────────────────┐
-│ files │ project (even)       │ project (references) │ service (even)       │ service (references) │
-┼───────┼──────────────────────┼──────────────────────┼──────────────────────┼──────────────────────┤
-│ 128   │ '1.149 s ±  0.030 s' │ '1.135 s ±  0.008 s' │ '1.178 s ±  0.010 s' │ '1.736 s ±  0.012 s' │
-│ 512   │ '1.636 s ±  0.009 s' │ '1.656 s ±  0.004 s' │ '1.895 s ±  0.007 s' │ '2.613 s ±  0.020 s' │
-│ 1024  │ '2.353 s ±  0.013 s' │ '2.399 s ±  0.016 s' │ '3.130 s ±  0.017 s' │ '4.034 s ±  0.061 s' │
-┴───────┴──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┘
+┌───────┬───────────────────────┬───────────────────────┐
+│ files │ project (even layout) │ service (even layout) │
+┼───────┼───────────────────────┼───────────────────────┤
+│ 1024  │ '1.750 s ±  0.008 s'  │ '2.473 s ±  0.011 s'  │
+┴───────┴───────────────────────┴───────────────────────┘
 ```
 
-See [typescript-eslint/typescript-eslint#9571 Performance: parserOptions.projectService no longer outperforms parserOptions.project](https://github.com/typescript-eslint/typescript-eslint/issues/9571)
+See [typescript-eslint/typescript-eslint#9571 Performance: parserOptions.projectService no longer outperforms parserOptions.project](https://github.com/typescript-eslint/typescript-eslint/issues/9571) in typescript-eslint.
+Also see the 📌 pinned issues later in this file.
 
 ### Result Measurement Notes
 

src/creators/files/createESLintConfigFile.ts

@@ -19,7 +19,7 @@ export function createESLintConfigFile({
 				files: ["**/*.ts"],
 				languageOptions: {
 					parserOptions: {
-						${singleRun ? "disallowAutomaticSingleRunInference: true," : ""}
+						${types !== "projectService" && !singleRun ? "disallowAutomaticSingleRunInference: true," : ""}
 						${projectKey}: ${typeof projectValue === "string" ? `"${projectValue}"` : projectValue},
 						tsconfigRootDir: import.meta.dirname,
 					},

src/data.ts

@@ -3,16 +3,15 @@ export const casesPath = "cases";
 export const caseEntries = [
 	{
 		label: "files",
-		values: [128, 512, 1024],
+		values: [1024],
 	},
 	{
 		label: "layout",
-		// values: ["even" /* , "references" , "wide" */],
-		values: ["even", "references" /* , "wide" */],
+		values: ["even"],
 	},
 	{
 		label: "singleRun",
-		values: [/* false, */ true],
+		values: [false],
 	},
 	{
 		label: "types",
@@ -22,9 +21,12 @@ export const caseEntries = [
 
 export type CaseEntry = (typeof caseEntries)[number];
 
-export type CaseData = {
-	[K in CaseEntry["label"]]: ({ label: K } & CaseEntry)["values"][number];
-};
+export interface CaseData {
+	files: number;
+	layout: "even" | "references" | "wide";
+	singleRun: boolean;
+	types: "project" | "service";
+}
 
 export interface NamedCaseData extends CaseData {
 	name: string;

src/generate.ts

@@ -41,10 +41,12 @@ const cases: NamedCaseData[] = [];
 
 for (const files of caseEntries[0].values) {
 	for (const layout of caseEntries[1].values) {
-		for (const types of caseEntries[3].values) {
-			const data: CaseData = { files, layout, singleRun: true, types };
-			const name = createProjectName(data);
-			cases.push(await createCase({ ...data, name }));
+		for (const singleRun of caseEntries[2].values) {
+			for (const types of caseEntries[3].values) {
+				const data: CaseData = { files, layout, singleRun, types };
+				const name = createProjectName(data);
+				cases.push(await createCase({ ...data, name }));
+			}
 		}
 	}
 }

src/measure.ts

@@ -37,40 +37,18 @@ const results: unknown[] = [];
 for (const files of caseEntries[0].values) {
 	results.push({
 		files,
-		"project (even)": await runProjectLint({
+		"project (even layout)": await runProjectLint({
 			files,
 			layout: "even",
-			singleRun: true,
+			singleRun: false,
 			types: "project",
 		}),
-		"project (references)": await runProjectLint({
-			files,
-			layout: "references",
-			singleRun: true,
-			types: "project",
-		}),
-		"service (even)": await runProjectLint({
+		"service (even layout)": await runProjectLint({
 			files,
 			layout: "even",
-			singleRun: true,
-			types: "service",
-		}),
-		"service (references)": await runProjectLint({
-			files,
-			layout: "references",
-			singleRun: true,
+			singleRun: false,
 			types: "service",
 		}),
-		// "project (wide)": await runProjectLint({
-		// 	files,
-		// 	layout: "wide",
-		// 	types: "project",
-		// }),
-		// "service (wide)": await runProjectLint({
-		// 	files,
-		// 	layout: "wide",
-		// 	types: "service",
-		// }),
 	});
 }