diff --git a/evaluation_function/correction/correction.py b/evaluation_function/correction/correction.py index bf5207a..9a8a0a6 100644 --- a/evaluation_function/correction/correction.py +++ b/evaluation_function/correction/correction.py @@ -33,11 +33,12 @@ def _check_minimality(fsa: FSA) -> Tuple[bool, Optional[ValidationError]]: try: minimized = hopcroft_minimization(fsa) if len(minimized.states) < len(fsa.states): + diff = len(fsa.states) - len(minimized.states) return False, ValidationError( - message=f"FSA is not minimal: has {len(fsa.states)} states but can be reduced to {len(minimized.states)}", + message=f"Your FSA works correctly, but it's not minimal! You have {len(fsa.states)} states, but only {len(minimized.states)} are needed. You could remove {diff} state(s).", code=ErrorCode.NOT_MINIMAL, severity="error", - suggestion="Minimize your FSA by merging equivalent states" + suggestion="Look for states that behave identically (same transitions and acceptance) - these can be merged into one" ) return True, None except Exception: @@ -69,9 +70,11 @@ def _build_feedback( hints = [e.suggestion for e in all_errors if e.suggestion] if structural_info: if structural_info.unreachable_states: - hints.append("Consider removing unreachable states") + unreachable = ", ".join(structural_info.unreachable_states) + hints.append(f"Tip: States {{{unreachable}}} can't be reached from your start state - you might want to remove them or add transitions to them") if structural_info.dead_states: - hints.append("Dead states can never lead to acceptance") + dead = ", ".join(structural_info.dead_states) + hints.append(f"Tip: States {{{dead}}} can never lead to acceptance - this might be intentional (trap states) or a bug") # Build language comparison language = LanguageComparison(are_equivalent=len(equivalence_errors) == 0) @@ -92,17 +95,20 @@ def _summarize_errors(errors: List[ValidationError]) -> str: for error in errors: msg = error.message.lower() if "alphabet" in msg: - error_types.add("alphabet mismatch") - elif "state" in msg and "count" in msg: - error_types.add("state count mismatch") - elif "accepting" in msg or "incorrectly marked" in msg: - error_types.add("acceptance error") - elif "transition" in msg: - error_types.add("transition error") + error_types.add("alphabet issue") + elif "states" in msg and ("many" in msg or "few" in msg or "needed" in msg): + error_types.add("incorrect number of states") + elif "accepting" in msg or "accept" in msg: + error_types.add("accepting states issue") + elif "transition" in msg or "reading" in msg: + error_types.add("transition issue") - if error_types: - return f"Languages differ: {', '.join(error_types)}" - return f"Languages differ: {len(errors)} issue(s)" + if len(error_types) == 1: + issue = list(error_types)[0] + return f"Almost there! Your FSA has an {issue}. Check the details below." + elif error_types: + return f"Your FSA doesn't quite match the expected language. Issues found: {', '.join(error_types)}" + return f"Your FSA doesn't accept the correct language. Found {len(errors)} issue(s) to fix." # ============================================================================= @@ -134,7 +140,11 @@ def analyze_fsa_correction( # Step 1: Validate student FSA structure student_errors = is_valid_fsa(student_fsa) if student_errors: - summary = "FSA has structural errors" + num_errors = len(student_errors) + if num_errors == 1: + summary = "Your FSA has a structural problem that needs to be fixed first. See the details below." + else: + summary = f"Your FSA has {num_errors} structural problems that need to be fixed first. See the details below." return Result( is_correct=False, feedback=summary, @@ -146,7 +156,7 @@ def analyze_fsa_correction( if expected_errors: return Result( is_correct=False, - feedback="Internal error: expected FSA is invalid" + feedback="Oops! There's an issue with the expected answer. Please contact your instructor." ) # Step 3: Check minimality if required @@ -162,15 +172,18 @@ def analyze_fsa_correction( equivalence_errors = fsas_accept_same_language(student_fsa, expected_fsa) if not equivalence_errors and not validation_errors: + # Success message with some stats + state_count = len(student_fsa.states) + feedback = f"Correct! Your FSA with {state_count} state(s) accepts exactly the right language. Well done!" return Result( is_correct=True, - feedback="Correct! FSA accepts the expected language.", - fsa_feedback=_build_feedback("FSA is correct", [], [], structural_info) + feedback=feedback, + fsa_feedback=_build_feedback("Your FSA is correct!", [], [], structural_info) ) # Build result with errors is_correct = len(equivalence_errors) == 0 and len(validation_errors) == 0 - summary = _summarize_errors(equivalence_errors) if equivalence_errors else "FSA has issues" + summary = _summarize_errors(equivalence_errors) if equivalence_errors else "Your FSA has some issues to address." return Result( is_correct=is_correct, diff --git a/evaluation_function/preview.py b/evaluation_function/preview.py index a47bcac..8331ec0 100755 --- a/evaluation_function/preview.py +++ b/evaluation_function/preview.py @@ -1,30 +1,290 @@ -from typing import Any +""" +Preview function for FSA validation. + +The preview function validates student FSA responses BEFORE submission. +It catches clear structural errors early, preventing students from submitting +invalid FSAs for full evaluation. + +Validation checks performed: +1. Parse check - Is the response a valid FSA structure? +2. Structural validation - Are states, initial, accept states, and transitions valid? +3. Warnings - Unreachable states, dead states, non-determinism (if applicable) +""" + +from typing import Any, List, Dict from lf_toolkit.preview import Result, Params, Preview -def preview_function(response: Any, params: Params) -> Result: +from .schemas import FSA, ValidationError +from .validation.validation import ( + is_valid_fsa, + is_deterministic, + find_unreachable_states, + find_dead_states, + get_structured_info_of_fsa, +) + + +def parse_fsa(value: Any) -> FSA: """ - Function used to preview a student response. - --- - The handler function passes three arguments to preview_function(): + Parse an FSA from various input formats. + + Args: + value: FSA as dict or JSON string + + Returns: + Parsed FSA object + + Raises: + ValueError: If the input cannot be parsed as a valid FSA + """ + if value is None: + raise ValueError("No FSA provided") + + if isinstance(value, str): + # Try to parse as JSON string + return FSA.model_validate_json(value) + elif isinstance(value, dict): + return FSA.model_validate(value) + else: + raise ValueError(f"Expected FSA as dict or JSON string, got {type(value).__name__}") - - `response` which are the answers provided by the student. - - `params` which are any extra parameters that may be useful, - e.g., error tolerances. - The output of this function is what is returned as the API response - and therefore must be JSON-encodable. It must also conform to the - response schema. +def format_errors_for_preview(errors: List[ValidationError], max_errors: int = 5) -> str: + """ + Format validation errors into a human-readable string for preview feedback. + + Args: + errors: List of ValidationError objects + max_errors: Maximum number of errors to show (to avoid overwhelming the user) + + Returns: + Formatted error string + """ + if not errors: + return "" + + # Separate errors by severity + critical_errors = [e for e in errors if e.severity == "error"] + warnings = [e for e in errors if e.severity == "warning"] + + lines = [] + + if critical_errors: + if len(critical_errors) == 1: + lines.append("There's an issue with your FSA that needs to be fixed:") + else: + lines.append(f"There are {len(critical_errors)} issues with your FSA that need to be fixed:") + lines.append("") + + for i, err in enumerate(critical_errors[:max_errors], 1): + lines.append(f" {i}. {err.message}") + if err.suggestion: + lines.append(f" >> {err.suggestion}") + lines.append("") + + if len(critical_errors) > max_errors: + lines.append(f" ... and {len(critical_errors) - max_errors} more issue(s)") + + if warnings: + if lines: + lines.append("") + lines.append("Some things to consider (not blocking, but worth checking):") + lines.append("") + for i, warn in enumerate(warnings[:max_errors], 1): + lines.append(f" - {warn.message}") + if warn.suggestion: + lines.append(f" >> {warn.suggestion}") + + if len(warnings) > max_errors: + lines.append(f" ... and {len(warnings) - max_errors} more suggestion(s)") + + return "\n".join(lines) - Any standard python library may be used, as well as any package - available on pip (provided it is added to requirements.txt). - The way you wish to structure you code (all in this function, or - split into many) is entirely up to you. +def errors_to_dict_list(errors: List[ValidationError]) -> List[Dict]: + """ + Convert ValidationError objects to dictionaries for JSON serialization. """ + return [ + { + "message": e.message, + "code": e.code.value if hasattr(e.code, 'value') else str(e.code), + "severity": e.severity, + "highlight": e.highlight.model_dump() if e.highlight else None, + "suggestion": e.suggestion + } + for e in errors + ] + +def preview_function(response: Any, params: Params) -> Result: + """ + Validate a student's FSA response before submission. + + This function performs structural validation to catch clear errors early, + preventing students from submitting obviously invalid FSAs for evaluation. + + Args: + response: Student's FSA response (dict or JSON string) + params: Extra parameters: + - require_deterministic (bool): Whether to require DFA (default: False) + - show_warnings (bool): Whether to show warnings (default: True) + + Returns: + Result with: + - preview.latex: FSA summary if valid + - preview.feedback: Error/warning messages if any + - preview.sympy: Structured validation data (errors, warnings, info) + """ + # Extract params with defaults + require_deterministic = False + show_warnings = True + + if hasattr(params, 'get'): + require_deterministic = params.get("require_deterministic", False) + show_warnings = params.get("show_warnings", True) + elif isinstance(params, dict): + require_deterministic = params.get("require_deterministic", False) + show_warnings = params.get("show_warnings", True) + try: - return Result(preview=Preview(sympy=response)) - except FeedbackException as e: - return Result(preview=Preview(feedback=str(e))) + # Step 1: Parse the FSA + fsa = parse_fsa(response) + except Exception as e: - return Result(preview=Preview(feedback=str(e))) + # Failed to parse - this is a critical error + error_msg = str(e) + + # Make error message more user-friendly + if "validation error" in error_msg.lower(): + if "states" in error_msg.lower(): + feedback = "Your FSA is missing the 'states' list. Every FSA needs a set of states to define!" + elif "alphabet" in error_msg.lower(): + feedback = "Your FSA is missing the 'alphabet'. What symbols should your automaton recognize?" + elif "initial_state" in error_msg.lower(): + feedback = "Your FSA needs an initial state - this is where processing begins!" + elif "transitions" in error_msg.lower(): + feedback = "There's an issue with your transitions. Each transition needs a from_state, to_state, and symbol." + else: + feedback = f"Your FSA structure isn't quite right: {error_msg}" + elif "json" in error_msg.lower(): + feedback = "Couldn't read your FSA data. Make sure it's properly formatted." + elif "no fsa" in error_msg.lower() or "none" in error_msg.lower(): + feedback = "No FSA provided! Please build your automaton before checking." + else: + feedback = f"There's a problem with your FSA format: {error_msg}" + + return Result( + preview=Preview( + feedback=feedback, + sympy={ + "valid": False, + "parse_error": True, + "errors": [{"message": feedback, "code": "PARSE_ERROR", "severity": "error"}] + } + ) + ) + + # Step 2: Structural validation + all_errors: List[ValidationError] = [] + + # Run structural validation (states, initial, accept, transitions) + structural_errors = is_valid_fsa(fsa) + all_errors.extend(structural_errors) + + # If there are structural errors, don't proceed with other checks + if structural_errors: + feedback = "Your FSA has some issues that need to be fixed before submission.\n\n" + feedback += format_errors_for_preview(all_errors) + return Result( + preview=Preview( + feedback=feedback, + sympy={ + "valid": False, + "errors": errors_to_dict_list(all_errors), + "num_states": len(fsa.states), + "num_transitions": len(fsa.transitions) + } + ) + ) + + # Step 3: Additional checks (determinism, unreachable states, dead states) + warnings: List[ValidationError] = [] + + # Check determinism if required + if require_deterministic: + det_errors = is_deterministic(fsa) + if det_errors: + all_errors.extend(det_errors) + + # Check for warnings (unreachable/dead states) + if show_warnings: + unreachable = find_unreachable_states(fsa) + dead = find_dead_states(fsa) + warnings.extend(unreachable) + warnings.extend(dead) + + # Get structural info + try: + info = get_structured_info_of_fsa(fsa) + info_dict = info.model_dump() + except Exception: + info_dict = { + "num_states": len(fsa.states), + "num_transitions": len(fsa.transitions), + "is_deterministic": len(is_deterministic(fsa)) == 0 + } + + # Step 4: Build response + has_errors = len(all_errors) > 0 + has_warnings = len(warnings) > 0 + + if has_errors: + # Critical errors - cannot submit + feedback = "Hold on! Your FSA has issues that need to be addressed.\n\n" + feedback += format_errors_for_preview(all_errors + warnings) + return Result( + preview=Preview( + feedback=feedback, + sympy={ + "valid": False, + "errors": errors_to_dict_list(all_errors), + "warnings": errors_to_dict_list(warnings), + **info_dict + } + ) + ) + + # Build success message + state_word = "state" if len(fsa.states) == 1 else "states" + trans_word = "transition" if len(fsa.transitions) == 1 else "transitions" + + fsa_type = "DFA (Deterministic)" if info_dict.get("is_deterministic") else "NFA (Non-deterministic)" + + summary = f"{fsa_type} with {len(fsa.states)} {state_word} and {len(fsa.transitions)} {trans_word}" + alphabet_str = ", ".join(f"'{s}'" for s in fsa.alphabet) + + if has_warnings: + # Valid but with warnings + warning_feedback = format_errors_for_preview(warnings) + feedback = f"Looking good! Your FSA is structurally valid.\n\n" + feedback += f"Summary: {summary}\n" + feedback += f"Alphabet: {{{alphabet_str}}}\n\n" + feedback += warning_feedback + else: + feedback = f"Great! Your FSA is structurally valid and ready for submission.\n\n" + feedback += f"Summary: {summary}\n" + feedback += f"Alphabet: {{{alphabet_str}}}" + + return Result( + preview=Preview( + latex=summary, # Short summary for display + feedback=feedback, + sympy={ + "valid": True, + "errors": [], + "warnings": errors_to_dict_list(warnings), + **info_dict + } + ) + ) diff --git a/evaluation_function/validation/validation.py b/evaluation_function/validation/validation.py index 0471cda..bd8b6d8 100644 --- a/evaluation_function/validation/validation.py +++ b/evaluation_function/validation/validation.py @@ -1,4 +1,3 @@ -from itertools import product from typing import Dict, List, Set from collections import deque @@ -21,10 +20,10 @@ def is_valid_fsa(fsa: FSA) -> List[ValidationError]: if not states: errors.append( ValidationError( - message="The FSA has no states defined", + message="Your FSA needs at least one state to work. Every automaton must have states to process input!", code=ErrorCode.EMPTY_STATES, severity="error", - suggestion="Add at least one state to the FSA" + suggestion="Start by adding a state - this will be your starting point for the automaton" ) ) return errors # Early return since other checks depend on states @@ -33,10 +32,10 @@ def is_valid_fsa(fsa: FSA) -> List[ValidationError]: if not alphabet: errors.append( ValidationError( - message="The alphabet is empty", + message="Your FSA needs an alphabet - the set of symbols it can read. Without an alphabet, there's nothing to process!", code=ErrorCode.EMPTY_ALPHABET, severity="error", - suggestion="Add at least one symbol to the alphabet" + suggestion="Define the input symbols your FSA should recognize (e.g., 'a', 'b', '0', '1')" ) ) @@ -44,14 +43,14 @@ def is_valid_fsa(fsa: FSA) -> List[ValidationError]: if fsa.initial_state not in states: errors.append( ValidationError( - message=f"The initial state '{fsa.initial_state}' is not defined in the FSA", + message=f"Oops! Your initial state '{fsa.initial_state}' doesn't exist in your FSA. The initial state must be one of your defined states.", code=ErrorCode.INVALID_INITIAL, severity="error", highlight=ElementHighlight( type="initial_state", state_id=fsa.initial_state ), - suggestion="Include the initial state in your FSA or change your initial state" + suggestion=f"Either add '{fsa.initial_state}' to your states, or choose an existing state as the initial state" ) ) @@ -60,14 +59,14 @@ def is_valid_fsa(fsa: FSA) -> List[ValidationError]: if acc not in states: errors.append( ValidationError( - message=f"The accept state '{acc}' is not defined in the FSA", + message=f"The accepting state '{acc}' isn't in your FSA. Accepting states must be part of your state set.", code=ErrorCode.INVALID_ACCEPT, severity="error", highlight=ElementHighlight( type="accept_state", state_id=acc ), - suggestion="Include the accept state in your FSA or change your accept state" + suggestion=f"Either add '{acc}' to your states, or remove it from accepting states" ) ) @@ -76,7 +75,7 @@ def is_valid_fsa(fsa: FSA) -> List[ValidationError]: if t.from_state not in states: errors.append( ValidationError( - message=f"The source state '{t.from_state}' in transition '{t.symbol}' is not defined", + message=f"This transition starts from '{t.from_state}', but that state doesn't exist in your FSA.", code=ErrorCode.INVALID_TRANSITION_SOURCE, severity="error", highlight=ElementHighlight( @@ -85,13 +84,13 @@ def is_valid_fsa(fsa: FSA) -> List[ValidationError]: to_state=t.to_state, symbol=t.symbol ), - suggestion=f"Add state '{t.from_state}' to the FSA or change the transition source" + suggestion=f"Add '{t.from_state}' to your states, or update this transition to start from an existing state" ) ) if t.to_state not in states: errors.append( ValidationError( - message=f"The destination state '{t.to_state}' in transition '{t.symbol}' is not defined", + message=f"This transition goes to '{t.to_state}', but that state doesn't exist in your FSA.", code=ErrorCode.INVALID_TRANSITION_DEST, severity="error", highlight=ElementHighlight( @@ -100,13 +99,13 @@ def is_valid_fsa(fsa: FSA) -> List[ValidationError]: to_state=t.to_state, symbol=t.symbol ), - suggestion=f"Add state '{t.to_state}' to the FSA or change the transition destination" + suggestion=f"Add '{t.to_state}' to your states, or update this transition to go to an existing state" ) ) if t.symbol not in alphabet: errors.append( ValidationError( - message=f"The transition symbol '{t.symbol}' is not in the alphabet", + message=f"The symbol '{t.symbol}' in this transition isn't in your alphabet. Transitions can only use symbols from the alphabet.", code=ErrorCode.INVALID_SYMBOL, severity="error", highlight=ElementHighlight( @@ -115,7 +114,7 @@ def is_valid_fsa(fsa: FSA) -> List[ValidationError]: to_state=t.to_state, symbol=t.symbol ), - suggestion=f"Add symbol '{t.symbol}' to the alphabet or change the transition symbol" + suggestion=f"Either add '{t.symbol}' to your alphabet, or change this transition to use an existing symbol" ) ) @@ -140,7 +139,7 @@ def is_deterministic(fsa: FSA) -> List[ValidationError]: if key in seen: errors.append( ValidationError( - message=f"Non-deterministic: multiple transitions from '{t.from_state}' on symbol '{t.symbol}'", + message=f"Your FSA has multiple transitions from state '{t.from_state}' when reading '{t.symbol}'. In a DFA, each state can only have one transition per symbol.", code=ErrorCode.DUPLICATE_TRANSITION, severity="error", highlight=ElementHighlight( @@ -149,7 +148,7 @@ def is_deterministic(fsa: FSA) -> List[ValidationError]: to_state=t.to_state, symbol=t.symbol ), - suggestion="Remove duplicate transitions or convert to NFA if nondeterminism is intended" + suggestion=f"Keep only one transition from '{t.from_state}' on '{t.symbol}', or if you meant to create an NFA, that's also valid!" ) ) seen.add(key) @@ -169,7 +168,7 @@ def is_complete(fsa: FSA) -> List[ValidationError]: errors.extend(det_errors) errors.append( ValidationError( - message="Cannot check completeness for non-deterministic FSA", + message="We can only check completeness for deterministic FSAs. Please fix the determinism issues first.", code=ErrorCode.NOT_DETERMINISTIC, severity="error" ) @@ -185,7 +184,7 @@ def is_complete(fsa: FSA) -> List[ValidationError]: if (state, symbol) not in transition_keys: errors.append( ValidationError( - message=f"Missing transition from state '{state}' on symbol '{symbol}' to make the FSA complete", + message=f"State '{state}' is missing a transition for symbol '{symbol}'. A complete DFA needs transitions for every symbol from every state.", code=ErrorCode.MISSING_TRANSITION, severity="error", highlight=ElementHighlight( @@ -193,7 +192,7 @@ def is_complete(fsa: FSA) -> List[ValidationError]: state_id=state, symbol=symbol ), - suggestion=f"Add a transition from state '{state}' on symbol '{symbol}'" + suggestion=f"Add a transition from '{state}' when reading '{symbol}' - it can go to any state, including a 'trap' state" ) ) return errors @@ -227,14 +226,14 @@ def find_unreachable_states(fsa: FSA) -> List[ValidationError]: if state not in visited: errors.append( ValidationError( - message=f"State '{state}' is unreachable from the initial state", + message=f"State '{state}' can never be reached! There's no path from your initial state to this state.", code=ErrorCode.UNREACHABLE_STATE, - severity="warning", # Changed to warning as it's not always an error + severity="warning", highlight=ElementHighlight( type="state", state_id=state ), - suggestion=f"Add a transition to state '{state}' from a reachable state, or remove it if unnecessary" + suggestion=f"Connect '{state}' to your FSA by adding a transition to it, or remove it if it's not needed" ) ) return errors @@ -253,14 +252,14 @@ def find_dead_states(fsa: FSA) -> List[ValidationError]: if state != fsa.initial_state or state not in fsa.accept_states: errors.append( ValidationError( - message=f"State '{state}' cannot reach any accepting state (no accept states defined)", + message=f"Your FSA has no accepting states, so no input string can ever be accepted! This means the language is empty.", code=ErrorCode.DEAD_STATE, severity="warning", highlight=ElementHighlight( type="state", state_id=state ), - suggestion="Add at least one accept state to the FSA" + suggestion="If you want your FSA to accept some strings, mark at least one state as accepting" ) ) return errors @@ -285,14 +284,14 @@ def find_dead_states(fsa: FSA) -> List[ValidationError]: if state not in reachable_to_accept: errors.append( ValidationError( - message=f"State '{state}' is dead (cannot reach any accepting state)", + message=f"State '{state}' is a dead end - once you enter it, you can never reach an accepting state. This is often called a 'trap state'.", code=ErrorCode.DEAD_STATE, - severity="warning", # Changed to warning as it's not always an error + severity="warning", highlight=ElementHighlight( type="state", state_id=state ), - suggestion=f"Add a transition from state '{state}' to a state that can reach an accept state, or make state '{state}' accepting" + suggestion=f"This might be intentional (to reject certain inputs), or you could add a path from '{state}' to an accepting state" ) ) return errors @@ -417,9 +416,6 @@ def get_structured_info_of_fsa(fsa: FSA) -> StructuralInfo: """ Get structured information about the FSA including properties and analysis. """ - # Get validation errors first - validation_errors = is_valid_fsa(fsa) - # Check determinism - returns boolean det_errors = is_deterministic(fsa) is_deterministic_bool = len(det_errors) == 0 @@ -460,25 +456,44 @@ def are_isomorphic(fsa1: FSA, fsa2: FSA) -> List[ValidationError]: errors = [] # 1. Alphabet Check (Mandatory) if set(fsa1.alphabet) != set(fsa2.alphabet): + student_only = set(fsa1.alphabet) - set(fsa2.alphabet) + expected_only = set(fsa2.alphabet) - set(fsa1.alphabet) + + msg_parts = ["Your alphabet doesn't match what's expected."] + if student_only: + msg_parts.append(f"You have extra symbols: {student_only}") + if expected_only: + msg_parts.append(f"You're missing symbols: {expected_only}") + errors.append( ValidationError( - message="The alphabet of your FSA does not match the required alphabet.", + message=" ".join(msg_parts), code=ErrorCode.LANGUAGE_MISMATCH, severity="error", - suggestion=f"Your alphabet: {set(fsa1.alphabet)}. Expected: {set(fsa2.alphabet)}." + suggestion="Make sure your alphabet contains exactly the symbols needed for this language" ) ) # 2. Basic Structural Check (State Count) if len(fsa1.states) != len(fsa2.states): - errors.append( - ValidationError( - message=f"FSA structure mismatch: expected {len(fsa2.states)} states, but found {len(fsa1.states)}.", - code=ErrorCode.LANGUAGE_MISMATCH, - severity="error", - suggestion="Verify if you have unnecessary states or if you have minimized your FSA." + if len(fsa1.states) > len(fsa2.states): + errors.append( + ValidationError( + message=f"Your FSA has {len(fsa1.states)} states, but the minimal solution only needs {len(fsa2.states)}. You might have redundant states.", + code=ErrorCode.LANGUAGE_MISMATCH, + severity="error", + suggestion="Look for states that behave identically and could be merged, or check for unreachable states" + ) + ) + else: + errors.append( + ValidationError( + message=f"Your FSA has {len(fsa1.states)} states, but at least {len(fsa2.states)} are needed. You might be missing some states.", + code=ErrorCode.LANGUAGE_MISMATCH, + severity="error", + suggestion="Think about what different 'situations' your FSA needs to remember - each usually needs its own state" + ) ) - ) # 3. State Mapping Initialization mapping: Dict[str, str] = {fsa1.initial_state: fsa2.initial_state} @@ -497,16 +512,26 @@ def are_isomorphic(fsa1: FSA, fsa2: FSA) -> List[ValidationError]: # 4. Check Acceptance Parity if (s1 in accept1) != (s2 in accept2): - expected_type = "accepting" if s2 in accept2 else "non-accepting" - errors.append( - ValidationError( - message=f"State '{s1}' is incorrectly marked. It should be an {expected_type} state.", - code=ErrorCode.LANGUAGE_MISMATCH, - severity="error", - highlight=ElementHighlight(type="state", state_id=s1), - suggestion=f"Toggle the 'accept' status of state '{s1}'." + if s2 in accept2: + errors.append( + ValidationError( + message=f"State '{s1}' should be an accepting state, but it's not marked as one. Strings that end here should be accepted!", + code=ErrorCode.LANGUAGE_MISMATCH, + severity="error", + highlight=ElementHighlight(type="state", state_id=s1), + suggestion=f"Mark state '{s1}' as an accepting state (add it to your accept states)" + ) + ) + else: + errors.append( + ValidationError( + message=f"State '{s1}' is marked as accepting, but it shouldn't be. Strings that end here should be rejected!", + code=ErrorCode.LANGUAGE_MISMATCH, + severity="error", + highlight=ElementHighlight(type="state", state_id=s1), + suggestion=f"Remove state '{s1}' from your accepting states" + ) ) - ) # 5. Check Transitions for every symbol in the shared alphabet for symbol in fsa1.alphabet: @@ -515,15 +540,26 @@ def are_isomorphic(fsa1: FSA, fsa2: FSA) -> List[ValidationError]: # Missing Transition Check if (dest1 is None) != (dest2 is None): - errors.append( - ValidationError( - message=f"Missing or extra transition from state '{s1}' on symbol '{symbol}'.", - code=ErrorCode.LANGUAGE_MISMATCH, - severity="error", - highlight=ElementHighlight(type="state", state_id=s1, symbol=symbol), - suggestion="Ensure your DFA is complete and follows the transition logic." + if dest1 is None: + errors.append( + ValidationError( + message=f"State '{s1}' is missing a transition for symbol '{symbol}'. What should happen when you read '{symbol}' here?", + code=ErrorCode.LANGUAGE_MISMATCH, + severity="error", + highlight=ElementHighlight(type="state", state_id=s1, symbol=symbol), + suggestion=f"Add a transition from '{s1}' on '{symbol}' to handle this input" + ) + ) + else: + errors.append( + ValidationError( + message=f"State '{s1}' has an unexpected transition on '{symbol}'. This transition might not be needed.", + code=ErrorCode.LANGUAGE_MISMATCH, + severity="error", + highlight=ElementHighlight(type="state", state_id=s1, symbol=symbol), + suggestion=f"Review if the transition from '{s1}' on '{symbol}' is correct" + ) ) - ) if dest1 is not None: if dest1 not in mapping: @@ -536,7 +572,7 @@ def are_isomorphic(fsa1: FSA, fsa2: FSA) -> List[ValidationError]: if mapping[dest1] != dest2: errors.append( ValidationError( - message=f"Transition from '{s1}' on '{symbol}' leads to the wrong state.", + message=f"When in state '{s1}' and reading '{symbol}', you go to '{dest1}', but that leads to incorrect behavior. Check where this transition should go!", code=ErrorCode.LANGUAGE_MISMATCH, severity="error", highlight=ElementHighlight( @@ -545,7 +581,7 @@ def are_isomorphic(fsa1: FSA, fsa2: FSA) -> List[ValidationError]: to_state=dest1, symbol=symbol ), - suggestion="Check if this transition should point to a different state." + suggestion=f"Think about what state the FSA should be in after reading '{symbol}' from '{s1}' - try tracing through some example strings" ) ) diff --git a/test_local.py b/test_local.py index 04804eb..01f3d2b 100644 --- a/test_local.py +++ b/test_local.py @@ -448,14 +448,268 @@ def run_test(test_num, test_name, response_data, answer_data, params=None): } ) +# ============================================================================= +# PREVIEW FUNCTION TESTS +# ============================================================================= + +print("\n") +print("#" * 70) +print("# PREVIEW FUNCTION TESTS (Pre-submission Validation)") +print("#" * 70) +print() + +from evaluation_function.preview import preview_function +from lf_toolkit.preview import Params as PreviewParams + + +def run_preview_test(test_num, test_name, response_data, params=None): + """Helper function to run a preview test case""" + print("=" * 70) + print(f"Preview Test {test_num}: {test_name}") + print("=" * 70) + + if params is None: + params = PreviewParams() + + try: + result = preview_function(response_data, params) + + # Handle both Result object and dict + if hasattr(result, 'to_dict'): + result_dict = result.to_dict() + elif isinstance(result, dict): + result_dict = result + else: + result_dict = {'preview': result} + + # Extract preview data + preview_data = result_dict.get('preview', {}) + if hasattr(preview_data, 'model_dump'): + preview_data = preview_data.model_dump() + + sympy_data = preview_data.get('sympy', {}) + is_valid = sympy_data.get('valid', False) if sympy_data else False + + status = "[VALID]" if is_valid else "[INVALID]" + print(f"{status} FSA is valid: {is_valid}") + + feedback = preview_data.get('feedback', '') + if feedback: + print(f" Feedback:\n {str(feedback).replace(chr(10), chr(10) + ' ')}") + + if sympy_data and sympy_data.get('errors'): + print(f" Errors: {len(sympy_data['errors'])}") + if sympy_data and sympy_data.get('warnings'): + print(f" Warnings: {len(sympy_data['warnings'])}") + + print() + return result_dict + except Exception as e: + print(f"[ERROR] {e}") + import traceback + traceback.print_exc() + print() + return None + + +# Preview Test P1: Valid DFA +run_preview_test( + "P1", "Valid DFA - should pass", + response_data={ + "states": ["q0", "q1"], + "alphabet": ["a", "b"], + "transitions": [ + {"from_state": "q0", "to_state": "q1", "symbol": "a"}, + {"from_state": "q1", "to_state": "q1", "symbol": "a"}, + {"from_state": "q1", "to_state": "q1", "symbol": "b"} + ], + "initial_state": "q0", + "accept_states": ["q1"] + } +) + +# Preview Test P2: Invalid initial state +run_preview_test( + "P2", "Invalid initial state - should fail", + response_data={ + "states": ["q0", "q1"], + "alphabet": ["a", "b"], + "transitions": [ + {"from_state": "q0", "to_state": "q1", "symbol": "a"} + ], + "initial_state": "q99", # Does not exist + "accept_states": ["q1"] + } +) + +# Preview Test P3: Invalid accept state +run_preview_test( + "P3", "Invalid accept state - should fail", + response_data={ + "states": ["q0", "q1"], + "alphabet": ["a", "b"], + "transitions": [ + {"from_state": "q0", "to_state": "q1", "symbol": "a"} + ], + "initial_state": "q0", + "accept_states": ["q99"] # Does not exist + } +) + +# Preview Test P4: Transition references non-existent state +run_preview_test( + "P4", "Transition to non-existent state - should fail", + response_data={ + "states": ["q0", "q1"], + "alphabet": ["a", "b"], + "transitions": [ + {"from_state": "q0", "to_state": "q99", "symbol": "a"} # q99 doesn't exist + ], + "initial_state": "q0", + "accept_states": ["q1"] + } +) + +# Preview Test P5: Transition with invalid symbol +run_preview_test( + "P5", "Transition with symbol not in alphabet - should fail", + response_data={ + "states": ["q0", "q1"], + "alphabet": ["a", "b"], + "transitions": [ + {"from_state": "q0", "to_state": "q1", "symbol": "c"} # 'c' not in alphabet + ], + "initial_state": "q0", + "accept_states": ["q1"] + } +) + +# Preview Test P6: FSA with unreachable states (warning) +run_preview_test( + "P6", "FSA with unreachable states - valid with warning", + response_data={ + "states": ["q0", "q1", "q2", "q3"], + "alphabet": ["a", "b"], + "transitions": [ + {"from_state": "q0", "to_state": "q1", "symbol": "a"}, + {"from_state": "q1", "to_state": "q1", "symbol": "a"}, + {"from_state": "q1", "to_state": "q1", "symbol": "b"}, + {"from_state": "q2", "to_state": "q3", "symbol": "a"} # q2, q3 unreachable + ], + "initial_state": "q0", + "accept_states": ["q1"] + } +) + +# Preview Test P7: FSA with dead states (warning) +run_preview_test( + "P7", "FSA with dead states - valid with warning", + response_data={ + "states": ["q0", "q1", "dead"], + "alphabet": ["a", "b"], + "transitions": [ + {"from_state": "q0", "to_state": "q1", "symbol": "a"}, + {"from_state": "q0", "to_state": "dead", "symbol": "b"}, + {"from_state": "q1", "to_state": "q1", "symbol": "a"}, + {"from_state": "q1", "to_state": "q1", "symbol": "b"}, + {"from_state": "dead", "to_state": "dead", "symbol": "a"}, + {"from_state": "dead", "to_state": "dead", "symbol": "b"} + ], + "initial_state": "q0", + "accept_states": ["q1"] # "dead" can never reach accept + } +) + +# Preview Test P8: Not a valid FSA structure (parse error) +run_preview_test( + "P8", "Invalid structure - missing required fields", + response_data={ + "states": ["q0"], + # Missing alphabet, transitions, initial_state, accept_states + } +) + +# Preview Test P9: Empty states list +run_preview_test( + "P9", "Empty states list - should fail", + response_data={ + "states": [], + "alphabet": ["a"], + "transitions": [], + "initial_state": "q0", + "accept_states": [] + } +) + +# Preview Test P10: NFA (valid, non-deterministic) +run_preview_test( + "P10", "Valid NFA - non-deterministic allowed", + response_data={ + "states": ["q0", "q1", "q2"], + "alphabet": ["a", "b"], + "transitions": [ + {"from_state": "q0", "to_state": "q1", "symbol": "a"}, + {"from_state": "q0", "to_state": "q2", "symbol": "a"}, # Non-deterministic + {"from_state": "q1", "to_state": "q1", "symbol": "b"}, + {"from_state": "q2", "to_state": "q2", "symbol": "b"} + ], + "initial_state": "q0", + "accept_states": ["q1"] + } +) + +# Preview Test P11: Epsilon transitions +run_preview_test( + "P11", "Epsilon NFA - epsilon transitions", + response_data={ + "states": ["q0", "q1", "q2"], + "alphabet": ["a", "b"], + "transitions": [ + {"from_state": "q0", "to_state": "q1", "symbol": "a"}, + {"from_state": "q1", "to_state": "q2", "symbol": "b"} + ], + "initial_state": "q0", + "accept_states": ["q2"] + } +) + +# Preview Test P12: Null/None response +run_preview_test( + "P12", "Null response - should fail gracefully", + response_data=None +) + +# Preview Test P13: String response (JSON) +import json +run_preview_test( + "P13", "JSON string response - should parse", + response_data=json.dumps({ + "states": ["q0", "q1"], + "alphabet": ["a"], + "transitions": [ + {"from_state": "q0", "to_state": "q1", "symbol": "a"}, + {"from_state": "q1", "to_state": "q1", "symbol": "a"} + ], + "initial_state": "q0", + "accept_states": ["q1"] + }) +) + print("=" * 70) -print("ALL TESTS COMPLETED!") +print("ALL EVALUATION + PREVIEW TESTS COMPLETED!") print("=" * 70) print("\nRun with: python test_local.py") -print("These tests cover:") +print("\nEvaluation tests cover:") print(" - Basic DFA equivalence") print(" - Non-deterministic FSAs (NFAs)") -print(" - Epsilon transitions (ε-NFAs)") +print(" - Epsilon transitions") print(" - Edge cases (empty language, single state, unreachable states)") print(" - Complex patterns (ending with 'ab', divisibility by 3)") -print(" - Validation errors") +print("\nPreview tests cover:") +print(" - Valid FSA validation") +print(" - Invalid initial/accept states") +print(" - Invalid transitions (state/symbol)") +print(" - Unreachable and dead states (warnings)") +print(" - Parse errors (invalid structure, null input)") +print(" - NFA and epsilon transition support") +print(" - JSON string input parsing")