From ac0c6bcf18efc4854a0bedb8a6541721baeed03a Mon Sep 17 00:00:00 2001 From: prathamesh Date: Fri, 12 Jun 2020 07:42:28 +0530 Subject: [PATCH] Create test cases easily for C CPP While creating a C or CPP standard function based question, the standard test case will simply be the assert function of CPP. This replaces the writing of complete C code and print inputs and expected output for the user code evaluation. Now the creator simply needs to write the assert function. Example: For add function (adding two numbers), the test case will be: assert(add(1, 2) == 3) --- yaksh/cpp_code_evaluator.py | 63 ++++---- .../evaluator_tests/test_c_cpp_evaluation.py | 135 +----------------- yaksh/fixtures/demo_questions.zip | Bin 9957 -> 2512 bytes 3 files changed, 40 insertions(+), 158 deletions(-) diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py index d249c66d8..96ec15233 100644 --- a/yaksh/cpp_code_evaluator.py +++ b/yaksh/cpp_code_evaluator.py @@ -3,6 +3,7 @@ import os from os.path import isfile import subprocess +from textwrap import dedent # Local imports from .file_utils import copy_files, delete_files @@ -49,16 +50,42 @@ def set_file_paths(self): return user_output_path, ref_output_path + def append_test_case_to_answer(self): + content = dedent(''' + {0} + #include + int main(){{ + {1}; + return 0; + }} + '''.format(self.user_answer, self.test_case)) + return content + def get_commands(self, clean_ref_code_path, user_output_path, ref_output_path): - compile_command = 'g++ {0} -c -o {1}'.format( + compile_command = 'g++ {0} -c -o {1}'.format( self.submit_code_path, user_output_path) - compile_main = 'g++ {0} {1} -o {2}'.format( - clean_ref_code_path, user_output_path, - ref_output_path + compile_main = 'g++ {0} -o {1}'.format( + clean_ref_code_path, ref_output_path ) return compile_command, compile_main + def trim_error(self, error_type, error): + err = error_type + try: + error_lines = error.splitlines() + for e in error_lines: + if ':' in e: + if error_type == 'Assertion Error:': + err = '{0} \n {1}'.format(err, e.split(':')[-1]) + else: + err = '{0} \n {1}'.format(err, e.split(':', 1)[1]) + else: + err = '{0} \n {1}'.format(err, e) + except Exception: + return '{0} \n {1}'.format(err, main_err) + return err + def compile_code(self): if self.compiled_user_answer and self.compiled_test_code: return None @@ -67,7 +94,8 @@ def compile_code(self): self.test_code_path = self.create_submit_code_file('main.c') self.write_to_submit_code_file(self.submit_code_path, self.user_answer) - self.write_to_submit_code_file(self.test_code_path, self.test_case) + main_content = self.append_test_case_to_answer() + self.write_to_submit_code_file(self.test_code_path, main_content) clean_ref_code_path = self.test_code_path if self.file_paths: self.files = copy_files(self.file_paths) @@ -90,7 +118,6 @@ def compile_code(self): stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - self.compiled_test_code = self._run_command( self.compile_main, shell=True, @@ -141,31 +168,13 @@ def check_code(self): success, err = True, None mark_fraction = 1.0 if self.partial_grading else 0.0 else: - err = "{0} \n {1}".format(stdout, stderr) + err = self.trim_error('Assertion Error:', stderr) raise AssertionError(err) else: - err = "Test case Error:" - try: - error_lines = main_err.splitlines() - for e in error_lines: - if ':' in e: - err = "{0} \n {1}".format(err, e.split(":", 1)[1]) - else: - err = "{0} \n {1}".format(err, e) - except Exception: - err = "{0} \n {1}".format(err, main_err) + err = self.trim_error('Test case Error:', main_err) raise TestCaseError(err) else: - err = "Compilation Error:" - try: - error_lines = stdnt_stderr.splitlines() - for e in error_lines: - if ':' in e: - err = "{0} \n {1}".format(err, e.split(":", 1)[1]) - else: - err = "{0} \n {1}".format(err, e) - except Exception: - err = "{0} \n {1}".format(err, stdnt_stderr) + err = self.trim_error('Compilation Error:', stdnt_stderr) raise CompilationError(err) return success, err, mark_fraction diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index 14ed80893..6079abc24 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -18,40 +18,7 @@ def setUp(self): with open(self.f_path, 'wb') as f: f.write('2'.encode('ascii')) tmp_in_dir_path = tempfile.mkdtemp() - self.tc_data = dedent(""" - #include - #include - - extern int add(int, int); - - template - - void check(T expect, T result) - { - if (expect == result) - { - printf("Correct: Expected %d got %d ",expect,result); - } - else - { - printf("Incorrect: Expected %d got %d ",expect,result); - exit (1); - } - } - - int main(void) - { - int result; - result = add(0,0); - printf("Input submitted to the function: 0, 0"); - check(0, result); - result = add(2,3); - printf("Input submitted to the function: 2 3"); - check(5,result); - printf("All Correct"); - return 0; - } - """) + self.tc_data = 'assert(add(2, 3) == 5)' self.test_case_data = [{"test_case": self.tc_data, "test_case_type": "standardtestcase", "weight": 0.0 @@ -157,33 +124,7 @@ def test_infinite_loop(self): def test_file_based_assert(self): # Given self.file_paths = [(self.f_path, False)] - self.tc_data = dedent(""" - #include - #include - - extern int ans(); - - template - void check(T expect,T result) - { - if (expect == result) - { - printf("Correct: Expected %d got %d ",expect,result); - } - else - { - printf("Incorrect: Expected %d got %d ",expect,result); - exit (0); - } - } - - int main(void) - { - int result; - result = ans(); - check(50, result); - } - """) + self.tc_data = 'assert(ans() == 50)' self.test_case_data = [{"test_case": self.tc_data, "test_case_type": "standardtestcase", "weight": 0.0 @@ -218,40 +159,7 @@ def test_file_based_assert(self): def test_incorrect_testcase(self): # Given - self.tc_data = dedent(""" - #include - #include - - extern int add(int, int); - - template - - void check(T expect, T result) - { - if (expect == result) - { - printf("Correct: Expected %d got %d ",expect,result); - } - else - { - printf("Incorrect: Expected %d got %d ",expect,result); - exit (1); - } - } - - int main(void) - { - int result; - result = add(0,0); - printf("Input submitted to the function: 0, 0"); - check(0, result); - result = add(2,3); - printf("Input submitted to the function: 2 3"); - check(5,result) - printf("All Correct"); - return 0; - } - """) + self.tc_data = 'assert(add(0, 1) == 1' user_answer = dedent("""\ int add(int a, int b) { @@ -828,42 +736,7 @@ def test_assert_with_hook(self): # Given user_answer = "int add(int a, int b)\n{return a+b;}" - assert_test_case = dedent("""\ - #include - #include - - extern int add(int, int); - - template - - void check(T expect, T result) - { - if (expect == result) - { - printf("Correct: Expected %d got %d ", - expect,result); - } - else - { - printf("Incorrect: Expected %d got %d ", - expect,result); - exit (1); - } - } - - int main(void) - { - int result; - result = add(0,0); - printf("Input submitted 0, 0"); - check(0, result); - result = add(2,3); - printf("Input submitted 2 3"); - check(5,result); - printf("All Correct"); - return 0; - } - """) + assert_test_case = 'assert(add(2, 3) == 5)' hook_code = dedent("""\ def check_answer(user_answer): diff --git a/yaksh/fixtures/demo_questions.zip b/yaksh/fixtures/demo_questions.zip index 6b0f8526c0d223d667043437a1b378467d5d0d3f..81381b14146bf542e2a6c2bc6c442fe68dd2edcd 100644 GIT binary patch literal 2512 zcmZ|RXE+;*8VB$oL5-?)Ld|2V*`Nd^XloTYH6pY^Y%xm3-fFi-YaO-KZXKJ5acrth z&4?rRp;XLJ6m@&<{dS*wpZEE{|M$!L{RcOu19AZX07gJ9^dr2^_h^{w8UTP20{}Py z`~Y99^D}>UAMa-q+@y9ik}=W?qW|>>F@5NWM}-- z3mqA`1f~Aq9|j$>+bWZ?sW=>O)7A0K7f+11A$Sf-p8oIlA-xv;i(-9j#?FI}avu!L zK-R`CuzrpR8x`FQ`UetL9_tJdDwE0TJxTdBqcGDsUA~*hO{qytb@s!s5j9hnE=k3c zt*Ipv)DcW+i(Bc{KVF)sO#4z)sO2SgbJ}3olcbHY&iE^;5>4K`eWBGa)Ra6FM}UfV zGB@8Asdo&Bs6=*A>Dk_O(xWku3F{GqDD}NX)$I5L^w5-ryrpbrfwSS;U^N$JL7Ft{ zn%}c-f7HN;k*!C|cuk{QzKB)7G8BOs)Xo@SGoiM9Y-2gn@{?~m5bdSIy6l*sW#dM_ zvbHtiEd9vb=if~r%t~{ZRmfML8iI)&T+U~2o}0l=`V6i?z{|7xOTm|&w6G9f#sZd& zsoNbv5LNRK5#DfP_(T9zh?YmWhZpPS59?>R+&wA;$*@1>{=vz&&mN|+Y^Um46nK}$ z9VUc|Y3fRG7pR6;-+P%{4$c%cGZ%MfwN*t#gOsQ=9sSF zrO1z50fGk-3I*GgC+gwD2yOeQrdiFHn_m91HH1DE=ma8P4#ExZ!k$S^As;FF#71jq zQAr`fCfg&px<>fZ*i5x>}6wblG)@@Id@gCZo=Q;f@MDUHO+cgsb^@x@v$L{Yq-)x&% zzuaXwG=`gj(N)KCi6QJ@1HrGo+LObLMI?WsrWLLmo3(a*GZSdG>9Ek>p5V{6E;#1Md&B5%!ffcX zhH#fugjxo;d9AO<2!Qm!c4>PuE9;#DQ@fNx%-;Q1S3#&Qkw3f~7n0N;(s z@iNAWx?gJs9{_oIAj3=(CZnLkJ&*WM_BMTo98x_dKUoL7u|EC1sEMZL!)Uxl5Q>z9KxoA%*fZD zP3w1hYpY2%f|c>e$~h~ST$$KNt@uvhp-!9D+EcQlwUC=fB+OOBN^(z(pZ?e@k%ODP z>dtu^V?7%JshQw%gp=T;Y$?2UvT5ci(>ci1?fKA}_T)KhMGkcFJc&+o;8S@(u0pb~ zq$E78#m3a(tJ8ANc~Z+~9f(}s1FbKTUNjsVO%fu7kQZdQdHeg*78tV5)fYr8oi^7+ zciSt2m2ZpK+kz^q?26E*`krC%fwzB&=q&a{MLYJ~{hn?}XFR!|P0;KJ96wTT7JLk@vz-;Y)m%K>+RU5kJ)Q|)Q|ybPeM^NdC;V^%5J^*F zY>o?`SK9SP-m?~B*zlgy&R?F-76x7~>*mnas^2VmeYdkpvEU3R`Au%cgEs7vd6(5lU9aF*d=ez z@Od6LXPkvQ_bJqGvUZ#6++6FLN@jh5CUC|S#9HYP&k)TLsLxUfh{vzR4fPAh4W%|5 zvXZR$BFbYzL|r0X{^+d`CV|bzWr@PFARZYoL@l1jsNolpPL~E13yf@a&B9>`>1$)Q zJGP!8H>5!X%`0PcNCTU;M=zHh@J+@jl*~@>Nd`qJEYno6uN!-EVSENDp%D>y0~wRS;z3+$qjC>^BKol>#E(P zAfV*|{ufxXsnhb7v3puZU8vE4g10eSaQ?36Uu(i@EBgY*+l1;Gi9D|6XZkj#*2th9 z?H1f{Z$k2G&GW{qZ(WH2mA^slo>2IW3i_xEL1ez}KMtGb6O1M6sZ8-ZM=KVdtu~iS zy{S7)R6=TBTF{@A)T~yKRyC7S*nBMYW%M#Sufs@XGhE^Xh>7Y3Ah_t@#6baeN0pLL+z z7x4lUJ|COBbf%)cIw<5o(z|$%S1_F!TeJy6_uY-*CDTy}e@h8o;<(h*4B zOx)|~8s??Y_$~>utA_p9KP04%X!!MqB7D23qK7}ei0JdUaZ(#3{3P_+@C=4FF?qE z)p5!1Y5{L{IEf(5u#%(i+HhkUT0y%1M=YM7^a22Aex^D4pX+~u3-ljw{TH_2#vuBC QUTA&}@@G5zbQ=KhFDO;28vpABlBVUq#$myQh8wiLLt&4fVVs~b`d!~E3PxtJ>!w-M{@iqSW?_d7+ z^}jy<_phJdympQMe}!i+a-C*@iPMRnM~mU9io(HQ&`R?tQpxF<{yu0C-49})vbm;X z73P}El;T!)y3npUh%-IYNh?(GELXGQ@!~X_o4Doc)JuW|GjaW&BtfQ|^mc>2cy}w< zqp3-xWop9EEQ5GP`bDmIFbzI}DA%t)qaC8V^lX|a?`8bA(b>zLv+LJe3j;CIF>McT zw=&GhQ>h-e2E^Ydd^4#@Oyp!PdV;cNl0 zHA|GQCn`;YSsdv&o8*hoD1S^-6{fls1@R;cB8{UeNe7~^(eR|M+%!Y4 z+2wURSD=-O4vnRJUZtM#H8b2j+_O9hOn0{q_r|Q^-NR&0uG~$&Em+WQ`%fA+(f)MQ zlh!S1-l%sg_r#lP@5tF5kv@Trz+!_@*fHyDLbd^sNFX0S1!X%Gi*ctqP6IYS=-a8R zFpCi$ai4mZ&UOb+WnI@8#=g(i2NUvw2sT23_k{cyDJ+kNGFSucoz%Y z+qFB7mh-@y(=rG{%G43ULad$X5xV8<>V|GR@O-Fbd&D0 zVo&;m=Bc|LtQPCpQV}7)5V$Qjg zlXIjKE43XVV!7E1=_2NgHpU!G5kUZ+dZRg}*blx+{F+5BbugR5+jA(ZjVrwvLWxe; z%hom1o{M+v(cQat{a{0tn|p_H#Bf7#TjXG6zb`Q&<*PqiF+^|gz+p>}@bz>>Gih&p zLDViBId9hibw=Ww%T0$zR^!-|_@3xvod7qSkLPUXUmLR^gC6wA2JRZ_xsp4u(1I+6 z!?s0uSDvRASxSlnxD7TOydfomIxaSt#l+XF8;Sbxs#S}IN zm6HVkq4p;-t8)9f@XBbywOgQ5bzlbAxb~9ROJFatz0A;~+MP8W5mvzu%pv0ST9*I= zDA}%vMHv}d5R^bO;JypxlQk{Hba%2i5Z;QpuoriiHtyVnT-&VC#@w$uy}2<*6E@Xy zJ$XwBj+>}BUFxKp!~ePOJ%Nn8+ypnjxgP4cH%I6bnI1ShYUUUJgSja)j{I%@%}!IgTcri`X26?lVAYP%0a( znO8%t7{#ac`YfuFwUd}ubrysJS?#v&lf92H0c2b(Z804LQ%NlB(2Br=*hWc6QRO0J za}nNt3FgYN9MLp3jn%`tt`aMlxmZ(ifmle++TWWTAV9ml45SbW;6F8aBFj2#S3{aB zc$d-JDmzRsHW_t#O~`fhlEus09*n1zEKhE2-&5vpf3!uDG`K}@3V`qMhMKa z=ugrKY1#Y6WI%xEh;jq5iTXzDYOan^kAU-(gjmJMgW&@D6zhOD2ElBp=o2y2Lcg{V+9<4zmG01mp&@nAga`SIKw7Q(i2%F-GLT5MZnh0TG5=aN&h%M!|OsWh_q({Vv$Y?+JGM`zfe{b{kivxUb@bTM zJT?JbIOj{<$1aNYA7kY?DeVdIm-5N{;c>b)&@!N|_u3#WYm*cH! z@OWj78Z1Zso&HWaL^#@g3pHHGa@40C{H?P5d{ryO9QF|poo1mz$Mx(Tq!OD7ed{s> zj}rt`YUft$9g%;PO@gvz%y|MgR^Bq2m5Vzk*jY7`%N*3B@9qZDDv&q8t>~u#$p#-4l$O! z3(#{Fz*rJ#0*t}Mc(EL5ZN2zh3m412uZx)ga`cwcQfj5HdL*IPviqWRtDBKuO(LX7-iuly4Fm$R?3EUzM{CFBskMWMCVEf036~rbeqwU?mwjeLK zusrY;W^k!(f8||qP+%ZzVqaeg-2XcWF2~4f{h&0K!fY73g`WO*ddJjsY zU&Rs|%2sY_uQ2>dpuVirWU4&CWGT{h+xKuro>UX4I*+N{0}vj5Hrn3#-8M9FWdUS% z;8@Y7PfJOD3}HdT-Oi2M!<}iJIwCY5&g1yUkZlwZI&`C3$ zRV%XZ&u2B`DX`%wtXO+J?i23eDCN zuAY1<@kq#(`862Em9purTY@)&x3Vf{KbMMR(&j>SG4AqOcW)z%5D+)%{PsOoLv@G)2GjS(sgT#?8{nZ z;T}6Kk+S+3lR8^}uxSCX+pyN)v1Mg@9X$NtXCHsK`F%8bQ4Jse`3HYn^LE<7!(aT8 Rn?AtrKCb`7i)`1f{T~RdA$