Feature: On-target execution tests [WIP]

.gitignore

@@ -56,6 +56,9 @@ tools/llgo
 tools/polly
 # Sphinx build tree, if building in-source dir.
 docs/_build
+# AVR on-target test suite library
+utils/AVR/avrlit/libavrlit/lufa
+utils/AVR/avrlit/libavrlit/*/_build
 
 #==============================================================================#
 # Files created in tree by the Go bindings.

CMakeLists.txt

@@ -586,6 +586,7 @@ if( LLVM_INCLUDE_UTILS )
   add_subdirectory(utils/not)
   add_subdirectory(utils/llvm-lit)
   add_subdirectory(utils/yaml-bench)
+  add_subdirectory(utils/AVR/avrlit)
 else()
   if ( LLVM_INCLUDE_TESTS )
     message(FATAL_ERROR "Including tests when not building utils will not work.

test/CodeGen/AVR/execute_on_target_test.cpp

@@ -0,0 +1,114 @@
+// RUN: llvm-avrlit %s %p/add.ll %p/and.ll %p/xor.ll
+
+#include <avrlit.h>
+
+using namespace avrlit;
+
+//=== add.ll ==================================================================
+extern "C" {
+  i8  add8_reg_reg(i8, i8);
+  i8  add8_reg_imm(i8);
+  i8  add8_reg_increment(i8);
+  i16 add16_reg_reg(i16, i16);
+  i16 add16_reg_imm(i16);
+  i16 add16_reg_imm_subi(i16);
+  i32 add32_reg_reg(i32, i32);
+  i32 add32_reg_imm(i32);
+  i64 add64_reg_reg(i64, i64);
+  i64 add64_reg_imm(i64);
+}
+
+void test_add(test & t) {
+  t.plan(20, PSTR("ADD"));
+
+  t.ok(_(add8_reg_reg( 23, 42) ==  23 + 42));
+  t.ok(_(add8_reg_reg(-23, 42) == -23 + 42));
+
+  t.ok(_(add8_reg_imm(42) == 42 + 5));
+  t.ok(_(add8_reg_imm(23) == 23 + 5));
+
+  t.ok(_(add8_reg_increment(42) == 42 + 1));
+  t.ok(_(add8_reg_increment(23) == 23 + 1));
+
+  t.ok(_(add16_reg_reg(1234, 4321) == 1234 + 4321));
+  t.ok(_(add16_reg_reg(5678, 8765) == 5678 + 8765));
+
+  t.ok(_(add16_reg_imm(1234) == 1234 + 63));
+  t.ok(_(add16_reg_imm(8765) == 8765 + 63));
+
+  t.ok(_(add16_reg_imm_subi(1234) == 1234 + 123));
+  t.ok(_(add16_reg_imm_subi(8765) == 8765 + 123));
+
+  t.ok(_(add32_reg_reg(123456, 987654) == 123456 + 987654));
+  t.ok(_(add32_reg_reg(3, 5) == 3 + 5));
+
+  t.ok(_(add32_reg_imm(987654) == 987654 + 5));
+  t.ok(_(add32_reg_imm(5) == 5 + 5));
+
+  t.ok(_(add64_reg_reg(9876543210,  9988776655) == 
+                       9876543210 + 9988776655));
+  t.ok(_(add64_reg_reg(198, 126) == 198 +126));
+
+  t.ok(_(add64_reg_imm(9876543210) == 9876543210 + 5));
+  t.ok(_(add64_reg_imm(198) == 198 + 5));
+
+}
+
+//=== and.ll ==================================================================
+extern "C" {
+  i8  and8_reg_reg(i8, i8);
+  i8  and8_reg_imm(i8);
+  i16 and16_reg_reg(i16, i16);
+  i16 and16_reg_imm(i16);
+}
+
+void test_and(test & t) {
+  t.plan(8, PSTR("AND"));
+
+  t.ok(_(and8_reg_reg(23, 42) ==  (23 & 42)));
+  t.ok(_(and8_reg_reg(0xef, 0x2a) ==  (0xef & 0x2a)));
+
+  t.ok(_(and8_reg_imm(23) ==  (23 & 5)));
+  t.ok(_(and8_reg_imm(0xef) ==  (0xef & 5)));
+
+  t.ok(_(and16_reg_reg(2323, 4242) ==  (2323 & 4242)));
+  t.ok(_(and16_reg_reg(0xefff, 0x2aaa) ==  (0xefff & 0x2aaa)));
+
+  t.ok(_(and16_reg_imm(2342) ==  (2342 & 1234)));
+  t.ok(_(and16_reg_imm(0xefff) ==  (0xefff & 1234)));
+}
+
+//=== xor.ll ==================================================================
+extern "C" {
+  i8  xor8_reg_reg(i8, i8);
+  i16 xor16_reg_reg(i16, i16);
+  i32 xor32_reg_reg(i32, i32);
+  i64 xor64_reg_reg(i64, i64);
+}
+
+void test_xor(test & t) {
+  t.plan(8, PSTR("XOR"));
+
+  t.ok(_(xor8_reg_reg(23, 42) ==  23 ^ 42));
+  t.ok(_(xor8_reg_reg(0xff, 0xaa) ==  0xff ^ 0xaa));
+
+  t.ok(_(xor16_reg_reg(0xffff, 0xaaaa) ==  0xffff ^ 0xaaaa));
+  t.ok(_(xor16_reg_reg(0x5555, 0xaaaa) ==  0x5555 ^ 0xaaaa));
+
+  t.ok(_(xor32_reg_reg(0xffffffff,  0xaaaaaaaa) == 
+                       0xffffffff ^ 0xaaaaaaaa));
+  t.ok(_(xor32_reg_reg(0x55555555,  0xaaaaaaaa) ==  
+                       0x55555555 ^ 0xaaaaaaaa));
+
+  t.ok(_(xor64_reg_reg(0xffffffffffffffff,  0xaaaaaaaaaaaaaaaa) == 
+                       0xffffffffffffffff ^ 0xaaaaaaaaaaaaaaaa));
+  t.ok(_(xor64_reg_reg(0x5555555555555555,  0xaaaaaaaaaaaaaaaa) ==  
+                       0x5555555555555555 ^ 0xaaaaaaaaaaaaaaaa));
+}
+
+//=== Test Suite ==============================================================
+
+AVRLIT_TEST_SUITE() {
+  run(test_add, test_and, test_xor);
+}
+

test/CodeGen/AVR/lit.local.cfg

@@ -1,6 +1,26 @@
-config.suffixes = ['.ll']
+config.suffixes = ['.ll', '.cpp']
+
+import os, lit.TestRunner
+from lit.formats import ShTest
 
 targets = set(config.root.targets_to_build.split())
 if not 'AVR' in targets:
-    config.unsupported = True
+  config.unsupported = True
+
+if 'AVRLIT_PORT' in os.environ:
+  config.environment['AVRLIT_PORT'] = os.environ['AVRLIT_PORT']
+
+class AVRCodeGenTest(ShTest):
+  def __init__(self):
+    ShTest.__init__(self)
+
+  def execute(self, test, litConfig):
+    if test.getSourcePath().endswith('.cpp') and not 'AVRLIT_PORT' in os.environ:
+      return (lit.Test.UNSUPPORTED, 'AVRLIT_PORT environment variable is not set')
+
+    return ShTest.execute(self, test, litConfig)
+
+
+config.test_format = AVRCodeGenTest()
 
+# vim: filetype=python

utils/AVR/avrlit/CMakeLists.txt

@@ -0,0 +1,22 @@
+if (WIN32 AND NOT CYGWIN)
+  # The script needs suffix.py for multiprocess to find a main module.
+  set(suffix .py)
+endif ()
+
+set(llvm_avrlit_path ${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-avrlit${suffix})
+
+if(NOT "${CMAKE_CFG_INTDIR}" STREQUAL ".")
+  foreach(BUILD_MODE ${CMAKE_CONFIGURATION_TYPES})
+    string(REPLACE ${CMAKE_CFG_INTDIR} ${BUILD_MODE} bi ${llvm_avrlit_path})
+    configure_file(
+      llvm-avrlit.in
+      ${bi}
+      )
+  endforeach()
+else()
+  set(BUILD_MODE .)
+  configure_file(
+    llvm-avrlit.in
+    ${llvm_avrlit_path}
+    )
+endif()

utils/AVR/avrlit/README.md

@@ -0,0 +1,61 @@
+# AVR LLVM Integrated Tester
+
+This tool builds an AVR executable from test code lowered by our backend and the
+`libavrlit` test suite library using a known good toolchain (avr-gcc). The
+resulting binary is uploaded to a development board. Test results are collected
+using a virtual tty.
+
+### Setup
+
+Things you will need:
+
+  * ATmega32U4 board with USB and AVR-109 type bootloader. (Arduino Leonardo,
+    RedBear Blend (Micro), &c.)
+  * [`pySerial`](http://pyserial.sourceforge.net) python module
+  * avr-gcc
+  * avrdude
+  * GNU Make
+  * Fire extinguisher
+
+Set the `AVRLIT_PORT` environment variable to the tty path of your board.
+
+```` bash
+> export AVRLIT_PORT=/dev/tty.usbmodemfd141
+````
+
+If your board currently runs an Arduino sketch that uses the serial port, you are
+all set. Otherwise, you need to reset the board manually for the first run. 
+
+```` bash
+> bin/llvm-lit -v ../llvm/test/CodeGen/AVR/
+```
+
+### Writing Tests
+
+The on-target execution tests reside in `llvm/test/CodeGen/AVR`. Like other lit 
+tests they contain a `RUN: ` line calling `llvm-avrlit`:
+
+```` C++
+// RUN: llvm-avrlit %s %p/add.ll 
+
+#include <avrlit.h>
+
+using namespace avrlit;
+
+extern "C" {  // actually this is extern "IR" but Bjarne forgot.
+  i8 add8_reg_reg(i8, i8);
+}
+
+AVRLIT_TEST(llvm_avr) {
+  reenter (this) {  // don't worry about the coroutine
+    plan(3);
+    ok(_(add8_reg_reg( 23, 42) ==  23 + 42)); yield;
+    ok(_(add8_reg_reg(-23, 42) == -23 + 42)); yield;
+    ok(_(add8_reg_reg(-23, 42) == 0));        yield;
+  }
+}
+````
+
+All of this is still in flux. I'll explain it if I decide to keep it. ;)
+
+

utils/AVR/avrlit/libavrlit/Makefile

@@ -0,0 +1,7 @@
+all:
+	$(MAKE) -C leonardo
+
+
+clean:
+	$(MAKE) -C leonardo clean
+

utils/AVR/avrlit/libavrlit/README.md

@@ -0,0 +1,22 @@
+# AVR LLVM Integrated Tester
+
+This directory contains the sources of `libavrlit`, a test suite library to
+test code generated by our backend on the target. It uses the
+[LUFA USB library](http://www.fourwalledcubicle.com/LUFA.php)
+([GitHub](https://github.com/abcminiuser/lufa)) and its makefiles. This library
+and the test executbale are compiled using avr-gcc.
+
+The test executables built with this framework connect to the host as a USB CDC
+device. They wait for the terminal to become ready before executing the tests. Results
+are printed in a format similar to `llvm-lit`.
+
+### Installation
+
+To avoid clutter the current library binary is checked in. To build the library
+the LUFA library is required:
+
+````
+> git clone https://github.com/abcminiuser/lufa.git
+> make
+````
+