Support SGLang as Potential Backend for Evaluation

README.md

@@ -241,14 +241,14 @@ vLLM occasionally differs in output from Huggingface. We treat Huggingface as th
 ### Tensor + Data Parallel and Fast Offline Batching Inference with `SGLang`
 We support SGLang with its efficient offline batch inference. Its **[Fast Backend Runtime](https://docs.sglang.ai/index.html)** thanks to efficient serving with RadixAttention for prefix caching, jump-forward constrained decoding, overhead-free CPU scheduler, continuous batching, token attention (paged attention), tensor parallelism, FlashInfer kernels, chunked prefill, and quantization (FP8/INT4/AWQ/GPTQ).
 
-To use SGLang as the evaluation backend, please **install it in advance** via SGLang documents [here](https://docs.sglang.ai/start/install.html#install-sglang). 
+To use SGLang as the evaluation backend, please **install it in advance** via SGLang documents [here](https://docs.sglang.ai/start/install.html#install-sglang).
 > [!Tip]
 > Due to the installing method of [`Flashinfer`](https://docs.flashinfer.ai/)-- a fast attention kernel library, we don't include the dependencies of `SGLang` within [pyproject.toml](pyproject.toml). Note that the `Flashinfer` also has some requirements on `torch` version.
 
 SGLang's server arguments are slightly different from other backends, see [here](https://docs.sglang.ai/backend/server_arguments.html) for more information. We provide an example of the usage here:
 ```bash
 lm_eval --model sglang \
-    --model_args pretrained={model_name},tp_size={data_parallel_size},dp_size={tensor_parallel_size},dtype=auto,mem-fraction-static=0.9, \
+    --model_args pretrained={model_name},dp_size={data_parallel_size},tp_size={tensor_parallel_size},dtype=auto,mem-fraction-static=0.9, \
     --tasks gsm8k_cot \
     --batch_size auto
 ```

pyproject.toml

@@ -78,7 +78,6 @@ zeno = ["pandas", "zeno-client"]
 wandb = ["wandb>=0.16.3", "pandas", "numpy"]
 gptqmodel = ["gptqmodel>=1.0.9"]
 japanese_leaderboard = ["emoji==2.14.0", "neologdn==0.5.3", "fugashi[unidic-lite]", "rouge_score>=0.1.2"]
-sglang =["sglang>=0.4.2.post2"]
 all = [
     "lm_eval[anthropic]",
     "lm_eval[dev]",
@@ -99,7 +98,6 @@ all = [
     "lm_eval[zeno]",
     "lm_eval[wandb]",
     "lm_eval[japanese_leaderboard]",
-    "lm_eval[sglang]",
 ]
 
 [tool.ruff.lint]

tests/models/test_sglang.py

@@ -3,8 +3,9 @@
 import pytest
 import torch
 
-from lm_eval import tasks
+from lm_eval import evaluate, simple_evaluate, tasks
 from lm_eval.api.instance import Instance
+from lm_eval.tasks import get_task_dict
 
 
 task_manager = tasks.TaskManager()
@@ -60,3 +61,54 @@ def test_logliklihood_rolling(self) -> None:
         res = self.LM.loglikelihood_rolling(self.ROLLING)
         for x in res:
             assert isinstance(x, float)
+
+    # def test_simple_evaluate(self)-> None:
+    #     results = simple_evaluate(
+    #         model =self.LM,
+    #         tasks=["gsm8k"],
+    #         # num_fewshot=0,
+    #         task_manager=task_manager,
+    #         limit= 1,
+    #     )
+    #     print(results)
+
+    # def test_evaluate(self)-> None:
+    #     tasks=["gsm8k"]
+    #     task_dict = get_task_dict(tasks, task_manager)
+    #     results = evaluate(
+    #     lm=self.LM,
+    #     task_dict=task_dict,
+    #     limit= 1,
+    #     )
+    #     print(results)
+
+    # TODO(jinwei): find out the outpt differences for "gsm_8k" with simple_evalute() and evaluate(). There are some errors in parser as well.
+    def test_evaluator(self) -> None:
+        simple_results = simple_evaluate(
+            model=self.LM,
+            tasks=["arc_easy"],
+            task_manager=task_manager,
+            limit=1,
+        )
+        assert simple_results is not None, "simple_evaluate returned None"
+
+        task_dict = get_task_dict(["arc_easy"], task_manager)
+        evaluate_results = evaluate(
+            lm=self.LM,
+            task_dict=task_dict,
+            limit=1,
+        )
+        assert evaluate_results is not None, "evaluate returned None"
+
+        assert set(simple_results["results"].keys()) == set(
+            evaluate_results["results"].keys()
+        ), "Mismatch in task keys between simple_evaluate and evaluate"
+
+        for task in simple_results["results"]:
+            assert (
+                simple_results["results"][task] == evaluate_results["results"][task]
+            ), f"Mismatch in results for {task}"
+
+        print(
+            "✅ test_evaluator passed: simple_evaluate and evaluate results are identical."
+        )