Add args and PR #

Author: pamelafox

.github/workflows/evaluate.yaml

@@ -136,7 +136,8 @@ jobs:
 
       - name: Evaluate local RAG flow
         run: |
-          python evals/evaluate.py
+          PR_NUMBER=${{ github.event.pull_request.number }}
+          python evals/evaluate.py --targeturl=http://127.0.0.1:8000/chat --numquestions=2 --resultsdir=results/pr$PR_NUMBER
 
       - name: Upload server logs as build artifact
         uses: actions/upload-artifact@v4
@@ -154,7 +155,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: eval_result
-          path: ./src/api/evaluate/eval_results.jsonl
+          path: ./evals/results/pr${{ github.event.pull_request.number }}
 
       - name: GitHub Summary Step
         if: ${{ success() }}

evals/evaluate.py

@@ -1,3 +1,4 @@
+import argparse
 import logging
 import os
 from pathlib import Path
@@ -50,14 +51,20 @@ def get_openai_config() -> dict:
     )
     load_dotenv(".env", override=True)
 
+    parser = argparse.ArgumentParser(description="Run evaluation with OpenAI configuration.")
+    parser.add_argument("--targeturl", type=str, help="Specify the target URL.")
+    parser.add_argument("--resultsdir", type=str, help="Specify the results directory.")
+    parser.add_argument("--numquestions", type=int, help="Specify the number of questions.")
+
+    args = parser.parse_args()
+
     openai_config = get_openai_config()
-    # TODO: specify the localhost URL using argument
-    # TODO: specify the experiment name (based on PR number)
-    # TODO: Specify the num questions using arg CLI
 
     run_evaluate_from_config(
         working_dir=Path(__file__).parent,
         config_path="eval_config.json",
+        num_questions=args.num_questions,
+        target_url=args.target_url,
+        results_dir=args.results_dir,
         openai_config=openai_config,
-        num_questions=None,
     )