From 75e8b391e69a6ee742b0fe3c0d60646036ba7cac Mon Sep 17 00:00:00 2001 From: anupam-dewan Date: Wed, 10 Sep 2025 13:39:27 -0700 Subject: [PATCH 1/2] adding concurency feature for litellm --- examples/model_configs/litellm_model.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/model_configs/litellm_model.yaml b/examples/model_configs/litellm_model.yaml index b0cb25199..0c5756a16 100644 --- a/examples/model_configs/litellm_model.yaml +++ b/examples/model_configs/litellm_model.yaml @@ -2,6 +2,7 @@ model_parameters: model_name: "openai/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" provider: "openai" base_url: "https://router.huggingface.co/hf-inference/v1" + concurrent_requests: 10 # Configure the number of concurrent API requests generation_parameters: temperature: 0.5 max_new_tokens: 256 From 8060c92269d5856d633f79e3ea314c2dc22e931b Mon Sep 17 00:00:00 2001 From: anupam-dewan Date: Wed, 10 Sep 2025 14:30:43 -0700 Subject: [PATCH 2/2] adding concurrent_requests as acceptable feild --- src/lighteval/models/abstract_model.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/lighteval/models/abstract_model.py b/src/lighteval/models/abstract_model.py index 46487656e..a46d9a361 100644 --- a/src/lighteval/models/abstract_model.py +++ b/src/lighteval/models/abstract_model.py @@ -54,6 +54,10 @@ class ModelConfig(BaseModel, extra="forbid"): behavior and context for the model during evaluation. cache_dir (str): Directory to cache the model. Defaults to "~/.cache/huggingface/lighteval". + concurrent_requests (int): + Maximum number of concurrent API requests to execute in parallel. + Higher values can improve throughput for batch processing but may hit rate limits + or exhaust API quotas faster. Default is 10. Methods: from_path(path: str): @@ -83,6 +87,8 @@ class ModelConfig(BaseModel, extra="forbid"): generation_parameters: GenerationParameters = GenerationParameters() system_prompt: str | None = None cache_dir: str = "~/.cache/huggingface/lighteval" + concurrent_requests: int = 10 + @classmethod def from_path(cls, path: str):