feat: Added support for failing cases

scripts/run_compiler_tests.sh

@@ -5,29 +5,53 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 fail_count=0
 success_count=0
 
-# We currently cannot test compiler errors, since we assume all runs need an exit code > 0
-# We assume compiler errors throw error codes < 100 for now
-for file in "$SCRIPT_DIR"/../tests/lesma/*.les; do
-	name=$(basename -s .les "${file}")
-	echo "Testing ${name}"
-	"$SCRIPT_DIR"/../build/lesma run "${file}" &> /tmp/compile_output.log
-  test_jit_ret_value=$?
-	"$SCRIPT_DIR"/../build/lesma compile "${file}" &> /tmp/compile_output.log
-	test_aot_ret_value=$?
+test_compiler() {
+  local file="$1"
+  local mode="$2"
+  "$SCRIPT_DIR"/../build/lesma "${mode}" "${file}" &> /dev/null
+  return $?
+}
+
+test_case() {
+  local file="$1"
+  local expected_to_fail="$2"
+  name=$(basename -s .les "${file}")
+  printf "Testing %s\n" "${name}"
+
   test_expected_ret_value=$(head -n 1 "$file" | cut -b 6-)
-  if [ "$test_jit_ret_value" -ne "$test_expected_ret_value" ]; then
+
+  test_compiler "${file}" "run"
+  test_jit_ret_value=$?
+
+  test_compiler "${file}" "compile"
+  test_aot_ret_value=$?
+
+  if [ "$test_jit_ret_value" -ne "$test_expected_ret_value" ] && [ "$expected_to_fail" -eq 0 ]; then
+    fail_count=$((fail_count + 1))
+    printf "  Run failed, expected %d, got %d\n" "$test_expected_ret_value" "$test_jit_ret_value"
+  elif [ $test_aot_ret_value -ne 0 ] && [ "$expected_to_fail" -eq 0 ]; then
     fail_count=$((fail_count + 1))
-    echo "  Run failed, expected $test_expected_ret_value, got $test_jit_ret_value"
-  elif [ $test_aot_ret_value -ne 0 ]; then
+    printf "  Compilation failed, expected 0, got %d\n" "$test_aot_ret_value"
+  elif [ "$expected_to_fail" -eq 1 ] && [ "$test_jit_ret_value" -eq "$test_expected_ret_value" ]; then
     fail_count=$((fail_count + 1))
-    echo "  Compilation failed, expected 0, got $test_aot_ret_value"
+    printf "  Run succeeded but was expected to fail\n"
   else
     success_count=$((success_count + 1))
-    echo "  Run succeeded"
+    printf "  %s\n" "$( [ "$expected_to_fail" -eq 1 ] && echo "Fail" || echo "Run" ) succeeded"
   fi
+}
+
+# Test success cases
+for file in "$SCRIPT_DIR"/../tests/lesma/success/*.les; do
+  test_case "${file}" 0
+done
+
+# Test failure cases
+for file in "$SCRIPT_DIR"/../tests/lesma/failure/*.les; do
+  test_case "${file}" 1
 done
 
-echo "Tests:"
-echo "  fail:    ${fail_count}"
-echo "  success: ${success_count}"
-exit ${fail_count}
+printf "Tests:\n"
+printf "  fail:    %d\n" "${fail_count}"
+printf "  success: %d\n" "${success_count}"
+exit ${fail_count}

tests/lesma/enum_compare.les → tests/lesma/failure/enum_compare.les

@@ -1,4 +1,4 @@
-#ret=65
+#ret=101
 def extern exit(x: int)
 enum Color
     RED
@@ -10,6 +10,5 @@ enum Color2
     BLUE
     GREEN
 
-#var x: Color = Color.RED
 if Color.RED != Color2.RED
     exit(101)