Merge pull request #197 from Azure-Samples/safetyeval

A few fixes from safety eval running

Author: pamelafox

evals/safety_evaluation.py

@@ -106,15 +106,13 @@ async def run_simulator(target_url: str, max_simulations: int):
             else:
                 logger.info(f"Failing score from:\nQ: {query}\nA: {answer}\n{evaluator} score: {eval_score}")
             numeric_severity_score = eval_score[f"{evaluator}_score"]
-            if isinstance(numeric_severity_score, float):
+            if isinstance(numeric_severity_score, float) or isinstance(numeric_severity_score, int):
                 summary_scores[evaluator]["score_total"] += numeric_severity_score
 
     # Compute the overall statistics
     for evaluator in evaluators:
         if len(outputs) > 0:
-            summary_scores[evaluator]["mean_score"] = (
-                summary_scores[evaluator]["score_total"] / summary_scores[evaluator]["low_count"]
-            )
+            summary_scores[evaluator]["mean_score"] = summary_scores[evaluator]["score_total"] / len(outputs)
             summary_scores[evaluator]["low_rate"] = summary_scores[evaluator]["low_count"] / len(outputs)
 
     # Save summary scores

src/backend/fastapi_app/postgres_models.py

@@ -52,7 +52,7 @@ def to_str_for_embedding(self):
 table_name = Item.__tablename__
 
 index_ada002 = Index(
-    "hnsw_index_for_cosine_{table_name}_embedding_ada002",
+    f"hnsw_index_for_cosine_{table_name}_embedding_ada002",
     Item.embedding_ada002,
     postgresql_using="hnsw",
     postgresql_with={"m": 16, "ef_construction": 64},

src/backend/fastapi_app/query_rewriter.py

@@ -69,7 +69,7 @@ def extract_search_arguments(original_user_query: str, chat_completion: ChatComp
                 arg = json.loads(function.arguments)
                 # Even though its required, search_query is not always specified
                 search_query = arg.get("search_query", original_user_query)
-                if "price_filter" in arg and arg["price_filter"]:
+                if "price_filter" in arg and arg["price_filter"] and isinstance(arg["price_filter"], dict):
                     price_filter = arg["price_filter"]
                     filters.append(
                         {
@@ -78,7 +78,7 @@ def extract_search_arguments(original_user_query: str, chat_completion: ChatComp
                             "value": price_filter["value"],
                         }
                     )
-                if "brand_filter" in arg and arg["brand_filter"]:
+                if "brand_filter" in arg and arg["brand_filter"] and isinstance(arg["brand_filter"], dict):
                     brand_filter = arg["brand_filter"]
                     filters.append(
                         {

src/backend/fastapi_app/routes/api_routes.py

@@ -145,6 +145,7 @@ async def chat_handler(
         if isinstance(e, APIError) and e.code == "content_filter":
             return ERROR_FILTER
         else:
+            logging.exception("Exception while generating response: %s", e)
             return {"error": str(e)}