Add the forwarding server with updated documentation

README.md

@@ -139,6 +139,9 @@ It uses the `scripts/evaluation/generic/deferral_generate.sh` script with three
 2. Evaluate the accuracy and pick the best deferral threshold 
 3. Run Co-LLM generation with the optimal threshold. 
 
+We also need to run the `forward.js` script to start a javascript-based batch request forwarding server: `node forward.js`. 
+It will be terribly slow to batch send async requests to the `vllm` server in Python (during the threshold searching stage); so we made this javascript based forwarding server to speed up this process. 
+The dependency is very minimal -- you only need to install `express` and `axios` with `npm install express axios`.
 
 ## Acknowledgement 
 

forward.js

@@ -0,0 +1,35 @@
+const express = require('express');
+const axios = require('axios');
+const app = express();
+
+app.use(express.json());
+
+app.post('/batch_generate', async (req, res) => {
+    const { forward_url, prompt_token_id_list, n, top_p, top_k, temperature, max_tokens, stop, logprobs } = req.body;
+
+    console.log("forward_url", forward_url);
+    try {
+        const promises = prompt_token_id_list.map(prompt_token_id =>
+            axios.post(forward_url, {
+                "prompt_token_ids": prompt_token_id,
+                n,
+                top_p,
+                top_k,
+                temperature,
+                max_tokens,
+                stop,
+                logprobs,
+            })
+        );
+
+        const responses = await Promise.all(promises);
+        const combinedResults = responses.map(r => r.data);
+
+        res.json(combinedResults);
+    } catch (error) {
+        res.status(500).send(error.toString());
+    }
+});
+
+const PORT = process.env.PORT || 8003;
+app.listen(PORT, () => console.log(`Server is running on port ${PORT}`));