From 6b2b114a406ac6a881b1ad4177e3c8432682ef74 Mon Sep 17 00:00:00 2001
From: everythingfades <raymondguhonglei@163.com>
Date: Wed, 21 Jan 2026 23:04:36 +0000
Subject: [PATCH] feat: temp workaround for checkpoint 1

---
 evaluation_function/compareFSA_service.py | 37 -----------------------
 evaluation_function/evaluation.py         | 21 ++++++++++---
 2 files changed, 16 insertions(+), 42 deletions(-)
 delete mode 100644 evaluation_function/compareFSA_service.py

diff --git a/evaluation_function/compareFSA_service.py b/evaluation_function/compareFSA_service.py
deleted file mode 100644
index 31be923..0000000
--- a/evaluation_function/compareFSA_service.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# dummy file for shimmy tests
-
-import sys
-import json
-from typing import Any
-from .evaluation import evaluation_function
-from lf_toolkit.evaluation import Params
-
-def main():
-    for line in sys.stdin:
-        try:
-            # Parse request from Shimmy
-            req = json.loads(line)
-            response: Any = req["input"]["response"]
-            answer: Any = req["input"]["answer"]
-            params_dict: dict = req["input"].get("params", {})
-            params = Params(params_dict)
-
-            # Call your evaluation function
-            result = evaluation_function(response, answer, params)
-
-            # Convert LFResult to JSON
-            resp = {
-                "output": {
-                    "is_correct": result.is_correct,
-                    "feedback_items": result.feedback_items
-                }
-            }
-            print(json.dumps(resp), flush=True)
-
-        except Exception as e:
-            # Always return JSON even on error
-            resp = {"output": {"is_correct": False, "feedback_items": [("error", str(e))]}}
-            print(json.dumps(resp), flush=True)
-
-if __name__ == "__main__":
-    main()
diff --git a/evaluation_function/evaluation.py b/evaluation_function/evaluation.py
index f789f2b..69ca268 100755
--- a/evaluation_function/evaluation.py
+++ b/evaluation_function/evaluation.py
@@ -1,7 +1,9 @@
 from typing import Any
 from lf_toolkit.evaluation import Result as LFResult, Params
 
-from .schemas import FSA, FSAFrontend
+# note: this file is a temperary workaround, if the frontend -> backend communication succeed, fix this file
+
+from .schemas import FSA#, FSAFrontend
 from .schemas.result import Result
 from .correction import analyze_fsa_correction
 
@@ -13,6 +15,11 @@
 #         feedback_items=[("error", f"{payload}")]
 #     )
 
+def validate_fsa(value: str | dict) -> FSA:
+    if isinstance(value, str):
+        return FSA.model_validate_json(value)
+    return FSA.model_validate(value)
+
 def evaluation_function(
     response: Any,
     answer: Any,
@@ -31,11 +38,15 @@ def evaluation_function(
     """
     try:
         # Parse FSAs from input
-        student_fsa_ = FSAFrontend.model_validate(response)
-        expected_fsa_ = FSAFrontend.model_validate(answer)
+        # student_fsa_ = FSAFrontend.model_validate(response)
+        # expected_fsa_ = FSAFrontend.model_validate(answer)
+
+        # student_fsa = student_fsa_.from_flattened()
+        # expected_fsa = expected_fsa_.from_flattened()
 
-        student_fsa = student_fsa_.from_flattened()
-        expected_fsa = expected_fsa_.from_flattened()
+        # as a temporary workaround we assume the response and answer are all valid json strings
+        student_fsa = validate_fsa(response)
+        expected_fsa = validate_fsa(answer)
 
         
         # Get require_minimal from params if present