Single class unit test framework in Java

Question

This is a single class (削除) and dependency free (削除ここまで)¹ unit test framework for Java. The only assertion available is assertTrue (this can be used for all test cases).

(Utilizes feature of Java7 : Multi catch)

What it does

• Execute all methods with annotation Test. should fail if any arguments are needed.
• Calculate elapsed time using a given repeat count.
• Print state for each test case.
• Print number of successful test cases and total number of test cases.
• assertTrue must be called to be successful.

What I want reviewed

• Potential performance improvements.
• Potential flaws in coding style.
• Potential flaws in API.
• Or anything else related to this that might be appropriate.

package info.simpll.simpletest;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
public class Tester {
 private static int successCount = 0;
 private static int testCount = 0;
 //--------------------------------------
 //Test annonation
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.METHOD)
 public static @interface Test {
 public String name() default "unnamed"; // name of test
 public int repeat() default 1; // repeat count if calcTime==true
 public boolean calcTime() default false; // calculate Time
 }
 //--------------------------------------
 //---------------------------------------
 //Logger
 private static enum Logger {
 INFO(">"), ERROR("Error -->");
 private final String val;
 private Logger(String val) {
 this.val = val;
 }
 @Override
 public String toString() {
 return val;
 }
 public static void log(Logger level, CharSequence message) {
 System.out.printf("%s %s", level, message);
 System.out.println();
 }
 public static void info(CharSequence message) {
 log(INFO, message);
 }
 public static void error(CharSequence message) {
 log(ERROR, message);
 }
 }
 //----------------------------------------
 //----------------------------------------
 //Assertion Mechanism
 public static class SuccessException extends RuntimeException {
 }
 public static class FailException extends RuntimeException {
 }
 /**
 * state that a scenario is confidently true
 *
 * @param trueScenario this should be true to be success
 */
 public static void assertTrue(boolean trueScenario) {
 if (trueScenario) {
 throw new SuccessException();
 } else {
 throw new FailException();
 }
 }
 //-----------------------------------------
 /**
 * execute a single test case
 *
 * @param test test annotation object
 * @param method method
 * @param object instance
 * @return true only if SuccessException in InvocationTargetException
 */
 private static boolean executeTest(Test test, Method method, Object object) {
 boolean ret = false;
 Logger.info("------------------------");
 try {
 Logger.info("Test case:" + test.name());
 Logger.info("Method:" + method.getName());
 Logger.info("Class:" + object.getClass().getName());
 method.invoke(object);
 } catch (IllegalAccessException | IllegalArgumentException ex) {
 Logger.error("Problem when calling method : " + method.getName());
 ex.printStackTrace();
 } catch (InvocationTargetException ex) {
 if (ex.getCause() instanceof SuccessException) {
 ret = true;
 } else if (ex.getCause() instanceof FailException) {
 //do nothing
 } else {
 ex.getCause().printStackTrace();
 }
 }
 if (ret) {
 Logger.info("Test case success!");
 //now calculate time if needed
 if (test.calcTime()) {
 calculateTime(test, method, object);
 }
 } else {
 Logger.info("Test case failed!");
 }
 Logger.info("------------------------\n");
 return ret;
 }
 /**
 * calculate time by running a test n times
 *
 * @param test test annotation object
 * @param method method
 * @param object instance
 */
 private static void calculateTime(Test test, Method method, Object object) {
 if (test.repeat() <= 0) {
 Logger.error("Invalid repeat count");
 return;
 }
 Logger.info("");
 Logger.info("Time test started");
 Logger.info(String.format("Repeat count:%d", test.repeat()));
 long start = System.currentTimeMillis();
 for (int i = 0; i < test.repeat(); i++) {
 try {
 method.invoke(object);
 } catch (IllegalAccessException | IllegalArgumentException ex) {
 Logger.error("Problem when calling method : " + method.
 getName());
 ex.printStackTrace();
 return;
 } catch (InvocationTargetException ex) {
 if (!(ex.getCause() instanceof SuccessException)) {
 //not success
 Logger.error("Test failed, time testing halted");
 if (!(ex.getCause() instanceof FailException)) {
 ex.getCause().printStackTrace();
 }
 return;
 }
 }
 }
 Logger.info(String.format("Time elapsed:%d mili seconds", System.
 currentTimeMillis() - start));
 }
 private static void printHelp() {
 Logger.info("Welcome to Simple Test");
 Logger.info("usage: java info.simpll.simpletest.Tester"
 + " your.package.name.YourClass");
 }
 public static void main(String[] args) {
 if (args == null || args.length != 1) {
 printHelp();
 return;
 }
 try {
 Class clazz = Class.forName(args[0]);
 Object object = clazz.newInstance();
 Method[] methods = clazz.getMethods();
 for (Method method : methods) {
 Test test = method.getAnnotation(Test.class);
 if (test != null) {
 //we can test it
 testCount++;
 if (executeTest(test, method, object)) {
 successCount++;
 }
 }
 }
 Logger.info(String.format("Tests %d out of %d passed", successCount,
 testCount));
 } catch (ClassNotFoundException | InstantiationException |
 IllegalAccessException ex) {
 Logger.error("Class instantiation faild, see if package "
 + " name and class name are correct");
 }
 }
}

Input Unit Test Class

package dummy.pkg;
import info.simpll.simpletest.Tester;
import info.simpll.simpletest.Tester.Test;
public class TheTest {
 @Test(name = "one equals one", repeat = 50000, calcTime = true)
 public void testOneEqOne() {
 Tester.assertTrue(1 == 1);
 }
 @Test(name = "two not equal to one")
 public void testTwoNotOne() {
 Tester.assertTrue(2 != 1);
 }
 @Test
 public void testJustTrueUnnamed() {
 Tester.assertTrue(true);
 }
 @Test(name = "This should fail")
 public void testFail() {
 Tester.assertTrue(false);
 }
 @Test(name = "This should fail with exception")
 public void testFailException() {
 Tester.assertTrue(5 / 0 > 5);
 }
}

Output

> ------------------------
> Test case:unnamed
> Method:testJustTrueUnnamed
> Class:dummy.pkg.TheTest
> Test case success!
> ------------------------
> ------------------------
> Test case:This should fail
> Method:testFailException
> Class:dummy.pkg.TheTest
java.lang.ArithmeticException: / by zero
 at dummy.pkg.TheTest.testFailException(TheTest.java:57)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 at java.lang.reflect.Method.invoke(Method.java:483)
 at info.simpll.simpletest.Tester.executeTest(Tester.java:126)
 at info.simpll.simpletest.Tester.main(Tester.java:218)
> Test case failed!
> ------------------------
> ------------------------
> Test case:This should fail
> Method:testFail
> Class:dummy.pkg.TheTest
> Test case failed!
> ------------------------
> ------------------------
> Test case:two not equal to one
> Method:testTwoNotOne
> Class:dummy.pkg.TheTest
> Test case success!
> ------------------------
> ------------------------
> Test case:one equals one
> Method:testOneEqOne
> Class:dummy.pkg.TheTest
> Test case success!
> 
> Time test started
> Repeat count:50000
> Time elapsed:259 mili seconds
> ------------------------
> Tests 3 out of 5 passed

Related Links

GitHub

Other info

Execute as java info.simpll.simpletest.Tester dummy.pkg.TheTest

(packages should be there)

---

¹ Removed to avoid invalidation of h.j.k.'s answer

• cHao: Any testing framework could get by with just an assertTrue if it wanted. The reason for the others is partly to reduce boilerplate and make failure messages more informative. (assertTrue(1 == 2) can only see that the condition is false, so you'd need to add a failure message...whereas assertEqual(1, 2) can see -- and thus report -- the expected and actual values, which would be more helpful in determining the reason for the failure.)
• JaDogg: Note : This code was/is always dependency free, Just scratched it, so h.j.k.'s answer is valid. However janos has not made that assumption because "dependency free" part was there.
• JaDogg: Blogged about this. Java Reflection Basics

Answer 1

public static void main(String[] args) {
 if (args == null || args.length != 1) {

The args param is never null. You don't need the null check.

---

Throwing SuccessException to indicate success... You know it stinks. There gotta be a better way. Just delete it and try to make it work without.

Step 1: delete the SuccessException class. It just feels good doesn't it.

Step 2: rewrite assertTrue to throw only on failure:

public static void assertTrue(boolean trueScenario) {
 if (!trueScenario) {
 throw new FailException();
 }
}

Step 3: modify executeTest to interpret "no exception thrown" as success

private static boolean executeTest(Test test, Method method, Object object) {
 boolean ret = true;
 Logger.info("------------------------");
 try {
 Logger.info("Test case:" + test.name());
 Logger.info("Method:" + method.getName());
 Logger.info("Class:" + object.getClass().getName());
 method.invoke(object);
 } catch (IllegalAccessException | IllegalArgumentException ex) {
 Logger.error("Problem when calling method : " + method.getName());
 ex.printStackTrace();
 } catch (InvocationTargetException ex) {
 ret = false;
 if (ex.getCause() instanceof FailException) {
 Logger.info("Test case failed!");
 } else {
 ex.getCause().printStackTrace();
 }
 }
 if (ret) {
 Logger.info("Test case success!");
 if (test.calcTime()) {
 calculateTime(test, method, object);
 }
 }
 Logger.info("------------------------\n");
 return ret;
}

Step 4: modify calculateTime similarly, no exception means success

private static void calculateTime(Test test, Method method, Object object) {
 if (test.repeat() <= 0) {
 Logger.error("Invalid repeat count");
 return;
 }
 Logger.info("");
 Logger.info("Time test started");
 Logger.info(String.format("Repeat count:%d", test.repeat()));
 long start = System.currentTimeMillis();
 for (int i = 0; i < test.repeat(); i++) {
 try {
 method.invoke(object);
 } catch (IllegalAccessException | IllegalArgumentException ex) {
 Logger.error("Problem when calling method : " + method.getName());
 ex.printStackTrace();
 return;
 } catch (InvocationTargetException ex) {
 Logger.error("Test failed, time testing halted");
 if (!(ex.getCause() instanceof FailException)) {
 ex.getCause().printStackTrace();
 }
 return;
 }
 }
 Logger.info(String.format("Time elapsed:%d mili seconds", System.
 currentTimeMillis() - start));
}

After these steps your test code still works. A small remark, the correct command to run is:

# taking account of the maven classpath
java -cp target/classes info.simpll.simpletest.Tester dummy.pkg.TheTest

Answer 2

1. You are reinventing-the-wheel for the unit testing framework, are you intending to do so for the logging parts as well? Perhaps consider integrating with an external logging framework?

   • If you are implementing your logging framework, then you may want to re-evaluate how you are specifying the prefix. What I can see now is that you define a private field val as the prefix, and then rely on your Logger.toString() representation inside the static log() method to print it. Sounds like a roundabout way to me...

2. You have code duplication when you need to do timing benchmarks, since you are invoking the method once first, and then if it is successful invoke it n times. You can consider treating all tests to run for n=1 times, which should simplify the implementation. Timing can then be automatically taken for cases where n > 1, and that may mean you don't need the explicit boolean flag to indicate whether the timing is required.

3. It sounds weird to have a SuccessException, but I suppose there's no other way (I didn't study how existing unit testing frameworks work).