From e26742c62993892deffbd44e68f3769423330cbb Mon Sep 17 00:00:00 2001
From: CNugteren <web@cedricnugteren.nl>
Date: Sat, 20 Jun 2015 10:58:21 +0200
Subject: [PATCH 1/6] Added additional absolute error checking when testing

---
 include/internal/utilities.h |  4 ++++
 src/utilities.cc             | 12 +++++++++---
 test/correctness/testabc.cc  |  2 +-
 test/correctness/testabc.h   |  1 -
 test/correctness/testaxy.cc  |  2 +-
 test/correctness/testaxy.h   |  1 -
 test/correctness/tester.cc   | 23 +++++++++++------------
 test/correctness/tester.h    |  5 +++--
 test/correctness/testxy.cc   |  2 +-
 test/correctness/testxy.h    |  1 -
 10 files changed, 30 insertions(+), 23 deletions(-)

diff --git a/include/internal/utilities.h b/include/internal/utilities.h
index af04dfdb..600605c7 100644
--- a/include/internal/utilities.h
+++ b/include/internal/utilities.h
@@ -143,6 +143,10 @@ bool CheckArgument(const int argc, char *argv[], std::string &help, const std::s
 // Returns a random number to be used as a seed
 unsigned int GetRandomSeed();
 
+// Test/example data lower and upper limit
+constexpr auto kTestDataLowerLimit = -2.0;
+constexpr auto kTestDataUpperLimit = 2.0;
+
 // Populates a vector with random data
 template <typename T>
 void PopulateVector(std::vector<T> &vector);
diff --git a/src/utilities.cc b/src/utilities.cc
index 80cea852..3fc33502 100644
--- a/src/utilities.cc
+++ b/src/utilities.cc
@@ -182,8 +182,10 @@ unsigned int GetRandomSeed() {
 // Create a random number generator and populates a vector with samples from a random distribution
 template <typename T>
 void PopulateVector(std::vector<T> &vector) {
+  auto lower_limit = static_cast<T>(kTestDataLowerLimit);
+  auto upper_limit = static_cast<T>(kTestDataUpperLimit);
   std::mt19937 mt(GetRandomSeed());
-  std::uniform_real_distribution<T> dist(static_cast<T>(-2.0), static_cast<T>(2.0));
+  std::uniform_real_distribution<T> dist(lower_limit, upper_limit);
   for (auto &element: vector) { element = dist(mt); }
 }
 template void PopulateVector<float>(std::vector<float>&);
@@ -192,14 +194,18 @@ template void PopulateVector<double>(std::vector<double>&);
 // Specialized versions of the above for complex data-types
 template <>
 void PopulateVector(std::vector<float2> &vector) {
+  auto lower_limit = static_cast<float>(kTestDataLowerLimit);
+  auto upper_limit = static_cast<float>(kTestDataUpperLimit);
   std::mt19937 mt(GetRandomSeed());
-  std::uniform_real_distribution<float> dist(-2.0f, 2.0f);
+  std::uniform_real_distribution<float> dist(lower_limit, upper_limit);
   for (auto &element: vector) { element.real(dist(mt)); element.imag(dist(mt)); }
 }
 template <>
 void PopulateVector(std::vector<double2> &vector) {
+  auto lower_limit = static_cast<double>(kTestDataLowerLimit);
+  auto upper_limit = static_cast<double>(kTestDataUpperLimit);
   std::mt19937 mt(GetRandomSeed());
-  std::uniform_real_distribution<double> dist(-2.0, 2.0);
+  std::uniform_real_distribution<double> dist(lower_limit, upper_limit);
   for (auto &element: vector) { element.real(dist(mt)); element.imag(dist(mt)); }
 }
 
diff --git a/test/correctness/testabc.cc b/test/correctness/testabc.cc
index f2880f50..0d78a24d 100644
--- a/test/correctness/testabc.cc
+++ b/test/correctness/testabc.cc
@@ -133,7 +133,7 @@ void TestABC<T>::TestRegular(Arguments<T> &args, const std::string &name) {
                             auto index = (args.layout == Layout::kRowMajor) ?
                                           idm*args.c_ld + idn + args.c_offset:
                                           idn*args.c_ld + idm + args.c_offset;
-                            if (!TestSimilarity(r_result[index], s_result[index], kErrorMargin)) {
+                            if (!TestSimilarity(r_result[index], s_result[index])) {
                               errors++;
                             }
                           }
diff --git a/test/correctness/testabc.h b/test/correctness/testabc.h
index f1e9e7f0..e995ba5b 100644
--- a/test/correctness/testabc.h
+++ b/test/correctness/testabc.h
@@ -31,7 +31,6 @@ class TestABC: public Tester<T> {
   // Uses several variables from the Tester class
   using Tester<T>::context_;
   using Tester<T>::queue_;
-  using Tester<T>::kErrorMargin;
   using Tester<T>::kLayouts;
   using Tester<T>::kTransposes;
 
diff --git a/test/correctness/testaxy.cc b/test/correctness/testaxy.cc
index ed0b06ab..50591613 100644
--- a/test/correctness/testaxy.cc
+++ b/test/correctness/testaxy.cc
@@ -125,7 +125,7 @@ void TestAXY<T>::TestRegular(Arguments<T> &args, const std::string &name) {
                       auto errors = size_t{0};
                       for (auto idm=size_t{0}; idm<m_real; ++idm) {
                         auto index = idm*y_inc + y_offset;
-                        if (!TestSimilarity(r_result[index], s_result[index], kErrorMargin)) {
+                        if (!TestSimilarity(r_result[index], s_result[index])) {
                           errors++;
                         }
                       }
diff --git a/test/correctness/testaxy.h b/test/correctness/testaxy.h
index 927477ed..97155d69 100644
--- a/test/correctness/testaxy.h
+++ b/test/correctness/testaxy.h
@@ -31,7 +31,6 @@ class TestAXY: public Tester<T> {
   // Uses several variables from the Tester class
   using Tester<T>::context_;
   using Tester<T>::queue_;
-  using Tester<T>::kErrorMargin;
   using Tester<T>::kLayouts;
   using Tester<T>::kTransposes;
 
diff --git a/test/correctness/tester.cc b/test/correctness/tester.cc
index f55af842..5fd9da92 100644
--- a/test/correctness/tester.cc
+++ b/test/correctness/tester.cc
@@ -17,7 +17,6 @@
 #include <vector>
 #include <iostream>
 #include <cmath>
-#include <limits>
 
 namespace clblast {
 // =================================================================================================
@@ -168,34 +167,34 @@ void Tester<T>::TestEnd() {
 // Compares two floating point values and returns whether they are within an acceptable error
 // margin. This replaces GTest's EXPECT_NEAR().
 template <typename T>
-bool Tester<T>::TestSimilarity(const T val1, const T val2, const double margin) {
+bool Tester<T>::TestSimilarity(const T val1, const T val2) {
   const auto difference = std::fabs(val1 - val2);
 
   // Shortcut, handles infinities
   if (val1 == val2) {
     return true;
   }
-  // The values are zero or both are extremely close to it relative error is less meaningful
-  else if (val1 == 0 || val2 == 0 || difference < std::numeric_limits<T>::min()) {
-    return difference < (static_cast<T>(margin) * std::numeric_limits<T>::min());
+  // The values are zero or very small: the relative error is less meaningful
+  else if (val1 == 0 || val2 == 0 || difference < static_cast<T>(kErrorMarginAbsolute)) {
+    return (difference < static_cast<T>(kErrorMarginAbsolute));
   }
   // Use relative error
   else {
-    return (difference / (std::fabs(val1) + std::fabs(val2))) < static_cast<T>(margin);
+    return (difference / (std::fabs(val1)+std::fabs(val2))) < static_cast<T>(kErrorMarginRelative);
   }
 }
 
 // Specialisations for complex data-types
 template <>
-bool Tester<float2>::TestSimilarity(const float2 val1, const float2 val2, const double margin) {
-  auto real = Tester<float>::TestSimilarity(val1.real(), val2.real(), margin);
-  auto imag = Tester<float>::TestSimilarity(val1.imag(), val2.imag(), margin);
+bool Tester<float2>::TestSimilarity(const float2 val1, const float2 val2) {
+  auto real = Tester<float>::TestSimilarity(val1.real(), val2.real());
+  auto imag = Tester<float>::TestSimilarity(val1.imag(), val2.imag());
   return (real && imag);
 }
 template <>
-bool Tester<double2>::TestSimilarity(const double2 val1, const double2 val2, const double margin) {
-  auto real = Tester<double>::TestSimilarity(val1.real(), val2.real(), margin);
-  auto imag = Tester<double>::TestSimilarity(val1.imag(), val2.imag(), margin);
+bool Tester<double2>::TestSimilarity(const double2 val1, const double2 val2) {
+  auto real = Tester<double>::TestSimilarity(val1.real(), val2.real());
+  auto imag = Tester<double>::TestSimilarity(val1.imag(), val2.imag());
   return (real && imag);
 }
 
diff --git a/test/correctness/tester.h b/test/correctness/tester.h
index 934141d2..e6815fce 100644
--- a/test/correctness/tester.h
+++ b/test/correctness/tester.h
@@ -44,7 +44,8 @@ class Tester {
   static constexpr auto kStatusError = -1.0f;
 
   // Set the allowed error margin for floating-point comparisons
-  static constexpr auto kErrorMargin = 1.0e-2;
+  static constexpr auto kErrorMarginRelative = 1.0e-2;
+  static constexpr auto kErrorMarginAbsolute = 1.0e-10;
 
   // Constants holding start and end strings for terminal-output in colour
   const std::string kPrintError{"\x1b[31m"};
@@ -84,7 +85,7 @@ class Tester {
   void TestEnd();
 
   // Compares two floating point values for similarity. Allows for a certain relative error margin.
-  static bool TestSimilarity(const T val1, const T val2, const double margin);
+  static bool TestSimilarity(const T val1, const T val2);
 
   // Tests either an error count (should be zero) or two error codes (must match)
   void TestErrorCount(const size_t errors, const size_t size, const Arguments<T> &args);
diff --git a/test/correctness/testxy.cc b/test/correctness/testxy.cc
index 49ae5d7d..b88600b7 100644
--- a/test/correctness/testxy.cc
+++ b/test/correctness/testxy.cc
@@ -100,7 +100,7 @@ void TestXY<T>::TestRegular(Arguments<T> &args, const std::string &name) {
               auto errors = size_t{0};
               for (auto idn=size_t{0}; idn<n; ++idn) {
                 auto index = idn*y_inc + y_offset;
-                if (!TestSimilarity(r_result[index], s_result[index], kErrorMargin)) {
+                if (!TestSimilarity(r_result[index], s_result[index])) {
                   errors++;
                 }
               }
diff --git a/test/correctness/testxy.h b/test/correctness/testxy.h
index 32cd91fa..2a9dcab9 100644
--- a/test/correctness/testxy.h
+++ b/test/correctness/testxy.h
@@ -31,7 +31,6 @@ class TestXY: public Tester<T> {
   // Uses several variables from the Tester class
   using Tester<T>::context_;
   using Tester<T>::queue_;
-  using Tester<T>::kErrorMargin;
 
   // Uses several helper functions from the Tester class
   using Tester<T>::TestStart;

From dfbc3365312a8ee1c100d7659e5814548192b48d Mon Sep 17 00:00:00 2001
From: CNugteren <web@cedricnugteren.nl>
Date: Sat, 20 Jun 2015 11:26:01 +0200
Subject: [PATCH 2/6] Moved the argument parsing to the common Tester base
 class

---
 test/correctness/routines/xaxpy.cc | 10 ++--------
 test/correctness/routines/xgemm.cc | 10 ++--------
 test/correctness/routines/xgemv.cc | 10 ++--------
 test/correctness/routines/xsymm.cc | 10 ++--------
 test/correctness/testabc.cc        |  4 ++--
 test/correctness/testabc.h         |  2 +-
 test/correctness/testaxy.cc        |  4 ++--
 test/correctness/testaxy.h         |  2 +-
 test/correctness/tester.cc         | 10 +++++++---
 test/correctness/tester.h          |  6 +++++-
 test/correctness/testxy.cc         |  4 ++--
 test/correctness/testxy.h          |  2 +-
 12 files changed, 29 insertions(+), 45 deletions(-)

diff --git a/test/correctness/routines/xaxpy.cc b/test/correctness/routines/xaxpy.cc
index aa90766e..45dcf6bb 100644
--- a/test/correctness/routines/xaxpy.cc
+++ b/test/correctness/routines/xaxpy.cc
@@ -46,19 +46,13 @@ void XaxpyTest(int argc, char *argv[], const bool silent, const std::string &nam
     return static_cast<StatusCode>(status);
   };
 
-  // Selects the platform and device on which to test (command-line options)
-  auto help = std::string{"Options given/available:\n"};
-  const auto platform_id = GetArgument(argc, argv, help, kArgPlatform, size_t{0});
-  const auto device_id = GetArgument(argc, argv, help, kArgDevice, size_t{0});
-  if (!silent) { fprintf(stdout, "\n* %s\n", help.c_str()); }
-
-  // Initializes the other arguments relevant for this routine
+  // Initializes the arguments relevant for this routine
   auto args = Arguments<T>{};
   const auto options = std::vector<std::string>{kArgN, kArgXInc, kArgYInc,
                                                 kArgXOffset, kArgYOffset, kArgAlpha};
 
   // Creates a tester
-  TestXY<T> tester{platform_id, device_id, name, options, clblast_lambda, clblas_lambda};
+  TestXY<T> tester{argc, argv, silent, name, options, clblast_lambda, clblas_lambda};
 
   // Runs the tests
   const auto case_name = "default";
diff --git a/test/correctness/routines/xgemm.cc b/test/correctness/routines/xgemm.cc
index 2de17518..4129e17c 100644
--- a/test/correctness/routines/xgemm.cc
+++ b/test/correctness/routines/xgemm.cc
@@ -56,13 +56,7 @@ void XgemmTest(int argc, char *argv[], const bool silent, const std::string &nam
     return static_cast<StatusCode>(status);
   };
 
-  // Selects the platform and device on which to test (command-line options)
-  auto help = std::string{"Options given/available:\n"};
-  const auto platform_id = GetArgument(argc, argv, help, kArgPlatform, size_t{0});
-  const auto device_id = GetArgument(argc, argv, help, kArgDevice, size_t{0});
-  if (!silent) { fprintf(stdout, "\n* %s\n", help.c_str()); }
-
-  // Initializes the other arguments relevant for this routine
+  // Initializes the arguments relevant for this routine
   auto args = Arguments<T>{};
   const auto options = std::vector<std::string>{kArgM, kArgN, kArgK, kArgLayout,
                                                 kArgATransp, kArgBTransp,
@@ -70,7 +64,7 @@ void XgemmTest(int argc, char *argv[], const bool silent, const std::string &nam
                                                 kArgAOffset, kArgBOffset, kArgCOffset};
 
   // Creates a tester
-  TestABC<T> tester{platform_id, device_id, name, options, clblast_lambda, clblas_lambda};
+  TestABC<T> tester{argc, argv, silent, name, options, clblast_lambda, clblas_lambda};
 
   // Loops over the test-cases from a data-layout point of view
   for (auto &layout: tester.kLayouts) {
diff --git a/test/correctness/routines/xgemv.cc b/test/correctness/routines/xgemv.cc
index 69faeb64..1f484eb4 100644
--- a/test/correctness/routines/xgemv.cc
+++ b/test/correctness/routines/xgemv.cc
@@ -50,20 +50,14 @@ void XgemvTest(int argc, char *argv[], const bool silent, const std::string &nam
     return static_cast<StatusCode>(status);
   };
 
-  // Selects the platform and device on which to test (command-line options)
-  auto help = std::string{"Options given/available:\n"};
-  const auto platform_id = GetArgument(argc, argv, help, kArgPlatform, size_t{0});
-  const auto device_id = GetArgument(argc, argv, help, kArgDevice, size_t{0});
-  if (!silent) { fprintf(stdout, "\n* %s\n", help.c_str()); }
-
-  // Initializes the other arguments relevant for this routine
+  // Initializes the arguments relevant for this routine
   auto args = Arguments<T>{};
   const auto options = std::vector<std::string>{kArgM, kArgN, kArgLayout, kArgATransp,
                                                 kArgALeadDim, kArgXInc, kArgYInc,
                                                 kArgAOffset, kArgXOffset, kArgYOffset};
 
   // Creates a tester
-  TestAXY<T> tester{platform_id, device_id, name, options, clblast_lambda, clblas_lambda};
+  TestAXY<T> tester{argc, argv, silent, name, options, clblast_lambda, clblas_lambda};
 
   // Loops over the test-cases from a data-layout point of view
   for (auto &layout: tester.kLayouts) {
diff --git a/test/correctness/routines/xsymm.cc b/test/correctness/routines/xsymm.cc
index 14d6a3f7..d769177f 100644
--- a/test/correctness/routines/xsymm.cc
+++ b/test/correctness/routines/xsymm.cc
@@ -56,13 +56,7 @@ void XsymmTest(int argc, char *argv[], const bool silent, const std::string &nam
     return static_cast<StatusCode>(status);
   };
 
-  // Selects the platform and device on which to test (command-line options)
-  auto help = std::string{"Options given/available:\n"};
-  const auto platform_id = GetArgument(argc, argv, help, kArgPlatform, size_t{0});
-  const auto device_id = GetArgument(argc, argv, help, kArgDevice, size_t{0});
-  if (!silent) { fprintf(stdout, "\n* %s\n", help.c_str()); }
-
-  // Initializes the other arguments relevant for this routine
+  // Initializes the arguments relevant for this routine
   auto args = Arguments<T>{};
   const auto options = std::vector<std::string>{kArgM, kArgN, kArgLayout,
                                                 kArgSide, kArgTriangle,
@@ -70,7 +64,7 @@ void XsymmTest(int argc, char *argv[], const bool silent, const std::string &nam
                                                 kArgAOffset, kArgBOffset, kArgCOffset};
 
   // Creates a tester
-  TestABC<T> tester{platform_id, device_id, name, options, clblast_lambda, clblas_lambda};
+  TestABC<T> tester{argc, argv, silent, name, options, clblast_lambda, clblas_lambda};
 
   // Loops over the test-cases from a data-layout point of view
   for (auto &layout: tester.kLayouts) {
diff --git a/test/correctness/testabc.cc b/test/correctness/testabc.cc
index 0d78a24d..107ffb70 100644
--- a/test/correctness/testabc.cc
+++ b/test/correctness/testabc.cc
@@ -20,10 +20,10 @@ namespace clblast {
 
 // Constructor, initializes the base class tester and input data
 template <typename T>
-TestABC<T>::TestABC(const size_t platform_id, const size_t device_id,
+TestABC<T>::TestABC(int argc, char *argv[], const bool silent,
                     const std::string &name, const std::vector<std::string> &options,
                     const Routine clblast_lambda, const Routine clblas_lambda):
-    Tester<T>{platform_id, device_id, name, options},
+    Tester<T>{argc, argv, silent, name, options},
     clblast_lambda_(clblast_lambda),
     clblas_lambda_(clblas_lambda) {
 
diff --git a/test/correctness/testabc.h b/test/correctness/testabc.h
index e995ba5b..6472af42 100644
--- a/test/correctness/testabc.h
+++ b/test/correctness/testabc.h
@@ -57,7 +57,7 @@ class TestABC: public Tester<T> {
                                            CommandQueue&)>;
 
   // Constructor, initializes the base class tester and input data
-  TestABC(const size_t platform_id, const size_t device_id,
+  TestABC(int argc, char *argv[], const bool silent,
           const std::string &name, const std::vector<std::string> &options,
           const Routine clblast_lambda, const Routine clblas_lambda);
 
diff --git a/test/correctness/testaxy.cc b/test/correctness/testaxy.cc
index 50591613..3588573a 100644
--- a/test/correctness/testaxy.cc
+++ b/test/correctness/testaxy.cc
@@ -20,10 +20,10 @@ namespace clblast {
 
 // Constructor, initializes the base class tester and input data
 template <typename T>
-TestAXY<T>::TestAXY(const size_t platform_id, const size_t device_id,
+TestAXY<T>::TestAXY(int argc, char *argv[], const bool silent,
                     const std::string &name, const std::vector<std::string> &options,
                     const Routine clblast_lambda, const Routine clblas_lambda):
-    Tester<T>{platform_id, device_id, name, options},
+    Tester<T>{argc, argv, silent, name, options},
     clblast_lambda_(clblast_lambda),
     clblas_lambda_(clblas_lambda) {
 
diff --git a/test/correctness/testaxy.h b/test/correctness/testaxy.h
index 97155d69..9135e2ba 100644
--- a/test/correctness/testaxy.h
+++ b/test/correctness/testaxy.h
@@ -59,7 +59,7 @@ class TestAXY: public Tester<T> {
                                            CommandQueue&)>;
 
   // Constructor, initializes the base class tester and input data
-  TestAXY(const size_t platform_id, const size_t device_id,
+  TestAXY(int argc, char *argv[], const bool silent,
           const std::string &name, const std::vector<std::string> &options,
           const Routine clblast_lambda, const Routine clblas_lambda);
 
diff --git a/test/correctness/tester.cc b/test/correctness/tester.cc
index 5fd9da92..e6285f32 100644
--- a/test/correctness/tester.cc
+++ b/test/correctness/tester.cc
@@ -34,10 +34,11 @@ template <> const std::vector<Transpose> Tester<double2>::kTransposes = {Transpo
 // General constructor for all CLBlast testers. It prints out the test header to stdout and sets-up
 // the clBLAS library for reference.
 template <typename T>
-Tester<T>::Tester(const size_t platform_id, const size_t device_id,
+Tester<T>::Tester(int argc, char *argv[], const bool silent,
                   const std::string &name, const std::vector<std::string> &options):
-    platform_(Platform(platform_id)),
-    device_(Device(platform_, kDeviceType, device_id)),
+    help_("Options given/available:\n"),
+    platform_(Platform(GetArgument(argc, argv, help_, kArgPlatform, size_t{0}))),
+    device_(Device(platform_, kDeviceType, GetArgument(argc, argv, help_, kArgDevice, size_t{0}))),
     context_(Context(device_)),
     queue_(CommandQueue(context_, device_)),
     error_log_{},
@@ -49,6 +50,9 @@ Tester<T>::Tester(const size_t platform_id, const size_t device_id,
     tests_passed_{0},
     options_{options} {
 
+  // Prints the help message (command-line arguments)
+  if (!silent) { fprintf(stdout, "\n* %s\n", help_.c_str()); }
+
   // Prints the header
   fprintf(stdout, "* Running on OpenCL device '%s'.\n", device_.Name().c_str());
   fprintf(stdout, "* Starting tests for the %s'%s'%s routine. Legend:\n",
diff --git a/test/correctness/tester.h b/test/correctness/tester.h
index e6815fce..ffc02bbf 100644
--- a/test/correctness/tester.h
+++ b/test/correctness/tester.h
@@ -76,7 +76,7 @@ class Tester {
 
   // Creates an instance of the tester, running on a particular OpenCL platform and device. It
   // takes the routine's names as an additional parameter.
-  explicit Tester(const size_t platform_id, const size_t device_id,
+  explicit Tester(int argc, char *argv[], const bool silent,
                   const std::string &name, const std::vector<std::string> &options);
   ~Tester();
 
@@ -97,6 +97,9 @@ class Tester {
   // Retrieves a list of example scalars of the right type
   const std::vector<T> GetExampleScalars();
 
+  // The help-message
+  std::string help_;
+
   // The OpenCL objects (accessible by derived classes)
   Platform platform_;
   Device device_;
@@ -105,6 +108,7 @@ class Tester {
 
  private:
 
+
   // Internal methods to report a passed, skipped, or failed test
   void ReportPass();
   void ReportSkipped();
diff --git a/test/correctness/testxy.cc b/test/correctness/testxy.cc
index b88600b7..b2d6c147 100644
--- a/test/correctness/testxy.cc
+++ b/test/correctness/testxy.cc
@@ -20,10 +20,10 @@ namespace clblast {
 
 // Constructor, initializes the base class tester and input data
 template <typename T>
-TestXY<T>::TestXY(const size_t platform_id, const size_t device_id,
+TestXY<T>::TestXY(int argc, char *argv[], const bool silent,
                   const std::string &name, const std::vector<std::string> &options,
                   const Routine clblast_lambda, const Routine clblas_lambda):
-    Tester<T>{platform_id, device_id, name, options},
+    Tester<T>{argc, argv, silent, name, options},
     clblast_lambda_(clblast_lambda),
     clblas_lambda_(clblas_lambda) {
 
diff --git a/test/correctness/testxy.h b/test/correctness/testxy.h
index 2a9dcab9..c806de14 100644
--- a/test/correctness/testxy.h
+++ b/test/correctness/testxy.h
@@ -56,7 +56,7 @@ class TestXY: public Tester<T> {
                                            CommandQueue&)>;
 
   // Constructor, initializes the base class tester and input data
-  TestXY(const size_t platform_id, const size_t device_id,
+  TestXY(int argc, char *argv[], const bool silent,
          const std::string &name, const std::vector<std::string> &options,
          const Routine clblast_lambda, const Routine clblas_lambda);
 

From 3ea3ba2beeab837e8d7c533746bd621daf1c09bd Mon Sep 17 00:00:00 2001
From: CNugteren <web@cedricnugteren.nl>
Date: Sat, 20 Jun 2015 13:33:50 +0200
Subject: [PATCH 3/6] Distinguish between a short smoke test and a full test

---
 include/internal/utilities.h |  5 +++++
 src/utilities.cc             | 15 ++++++++++-----
 test/correctness/testabc.h   |  3 ++-
 test/correctness/testaxy.h   |  3 ++-
 test/correctness/tester.cc   | 20 ++++++++++++++++----
 test/correctness/tester.h    |  7 ++++++-
 test/correctness/testxy.h    |  3 ++-
 7 files changed, 43 insertions(+), 13 deletions(-)

diff --git a/include/internal/utilities.h b/include/internal/utilities.h
index 600605c7..4cefdc4a 100644
--- a/include/internal/utilities.h
+++ b/include/internal/utilities.h
@@ -64,6 +64,9 @@ constexpr auto kArgStepSize = "step";
 constexpr auto kArgNumSteps = "num_steps";
 constexpr auto kArgNumRuns = "runs";
 
+// The client-specific arguments in string form
+constexpr auto kArgFullTest = "full_test";
+
 // The common arguments in string form
 constexpr auto kArgPlatform = "platform";
 constexpr auto kArgDevice = "device";
@@ -105,6 +108,8 @@ struct Arguments {
   size_t step = 1;
   size_t num_steps = 0;
   size_t num_runs = 10;
+  // Tester-specific arguments
+  bool full_test = false;
   // Common arguments
   size_t platform_id = 0;
   size_t device_id = 0;
diff --git a/src/utilities.cc b/src/utilities.cc
index 3fc33502..98570088 100644
--- a/src/utilities.cc
+++ b/src/utilities.cc
@@ -159,16 +159,21 @@ Precision GetPrecision(const int argc, char *argv[]) {
 bool CheckArgument(const int argc, char *argv[], std::string &help,
                    const std::string &option) {
 
-  // Updates the help message
-  help += "    -"+option+"\n";
-
   // Parses the argument. Note that this supports both the given option (e.g. -device) and one with
   // an extra dash in front (e.g. --device).
+  auto return_value = false;
   for (int c=0; c<argc; ++c) {
     auto item = std::string{argv[c]};
-    if (item.compare("-"+option) == 0 || item.compare("--"+option) == 0) { ++c; return true; }
+    if (item.compare("-"+option) == 0 || item.compare("--"+option) == 0) {
+      ++c;
+      return_value = true;
+    }
   }
-  return false;
+
+  // Updates the help message and returns
+  help += "    -"+option+" ";
+  help += (return_value) ? "[true]\n" : "[false]\n";
+  return return_value;
 }
 
 // =================================================================================================
diff --git a/test/correctness/testabc.h b/test/correctness/testabc.h
index 6472af42..b7601024 100644
--- a/test/correctness/testabc.h
+++ b/test/correctness/testabc.h
@@ -41,10 +41,11 @@ class TestABC: public Tester<T> {
   using Tester<T>::TestErrorCount;
   using Tester<T>::TestErrorCodes;
   using Tester<T>::GetExampleScalars;
+  using Tester<T>::GetOffsets;
 
   // Test settings for the regular test. Append to this list in case more tests are required.
   const std::vector<size_t> kMatrixDims = { 7, 64 };
-  const std::vector<size_t> kOffsets = { 0 };
+  const std::vector<size_t> kOffsets = GetOffsets();
   const std::vector<T> kAlphaValues = GetExampleScalars();
   const std::vector<T> kBetaValues = GetExampleScalars();
 
diff --git a/test/correctness/testaxy.h b/test/correctness/testaxy.h
index 9135e2ba..00d4f6ec 100644
--- a/test/correctness/testaxy.h
+++ b/test/correctness/testaxy.h
@@ -41,10 +41,11 @@ class TestAXY: public Tester<T> {
   using Tester<T>::TestErrorCount;
   using Tester<T>::TestErrorCodes;
   using Tester<T>::GetExampleScalars;
+  using Tester<T>::GetOffsets;
 
   // Test settings for the regular test. Append to this list in case more tests are required.
   const std::vector<size_t> kMatrixVectorDims = { 61, 512 };
-  const std::vector<size_t> kOffsets = { 0, 10 };
+  const std::vector<size_t> kOffsets = GetOffsets();
   const std::vector<size_t> kIncrements = { 1, 2 };
   const std::vector<T> kAlphaValues = GetExampleScalars();
   const std::vector<T> kBetaValues = GetExampleScalars();
diff --git a/test/correctness/tester.cc b/test/correctness/tester.cc
index e6285f32..09b0b5bd 100644
--- a/test/correctness/tester.cc
+++ b/test/correctness/tester.cc
@@ -41,6 +41,7 @@ Tester<T>::Tester(int argc, char *argv[], const bool silent,
     device_(Device(platform_, kDeviceType, GetArgument(argc, argv, help_, kArgDevice, size_t{0}))),
     context_(Context(device_)),
     queue_(CommandQueue(context_, device_)),
+    full_test_(CheckArgument(argc, argv, help_, kArgFullTest)),
     error_log_{},
     num_passed_{0},
     num_skipped_{0},
@@ -261,19 +262,30 @@ void Tester<T>::TestErrorCodes(const StatusCode clblas_status, const StatusCode
 // routines. This function is specialised for the different data-types.
 template <>
 const std::vector<float> Tester<float>::GetExampleScalars() {
-  return {0.0f, 1.0f, 3.14f};
+  if (full_test_) { return {0.0f, 1.0f, 3.14f}; }
+  else { return {3.14f}; }
 }
 template <>
 const std::vector<double> Tester<double>::GetExampleScalars() {
-  return {0.0, 1.0, 3.14};
+  if (full_test_) { return {0.0, 1.0, 3.14}; }
+  else { return {3.14}; }
 }
 template <>
 const std::vector<float2> Tester<float2>::GetExampleScalars() {
-  return {{0.0f, 0.0f}, {1.0f, 1.3f}, {2.42f, 3.14f}};
+  if (full_test_) { return {{0.0f, 0.0f}, {1.0f, 1.3f}, {2.42f, 3.14f}}; }
+  else { return {{2.42f, 3.14f}}; }
 }
 template <>
 const std::vector<double2> Tester<double2>::GetExampleScalars() {
-  return {{0.0, 0.0}, {1.0, 1.3}, {2.42, 3.14}};
+  if (full_test_) { return {{0.0, 0.0}, {1.0, 1.3}, {2.42, 3.14}}; }
+  else { return {{2.42, 3.14}}; }
+}
+
+// Retrieves the offset values to test with
+template <typename T>
+const std::vector<size_t> Tester<T>::GetOffsets() {
+  if (full_test_) { return {0, 10}; }
+  else { return {0}; }
 }
 
 // =================================================================================================
diff --git a/test/correctness/tester.h b/test/correctness/tester.h
index ffc02bbf..1085c472 100644
--- a/test/correctness/tester.h
+++ b/test/correctness/tester.h
@@ -97,6 +97,9 @@ class Tester {
   // Retrieves a list of example scalars of the right type
   const std::vector<T> GetExampleScalars();
 
+  // Retrieves a list of offset values to test
+  const std::vector<size_t> GetOffsets();
+
   // The help-message
   std::string help_;
 
@@ -108,7 +111,6 @@ class Tester {
 
  private:
 
-
   // Internal methods to report a passed, skipped, or failed test
   void ReportPass();
   void ReportSkipped();
@@ -117,6 +119,9 @@ class Tester {
   // Prints the error or success symbol to screen
   void PrintTestResult(const std::string &message);
 
+  // Whether or not to run the full test-suite or just a smoke test
+  bool full_test_;
+
   // Logging and counting occurrences of errors
   std::vector<ErrorLogEntry> error_log_;
   size_t num_passed_;
diff --git a/test/correctness/testxy.h b/test/correctness/testxy.h
index c806de14..8f2b6876 100644
--- a/test/correctness/testxy.h
+++ b/test/correctness/testxy.h
@@ -39,10 +39,11 @@ class TestXY: public Tester<T> {
   using Tester<T>::TestErrorCount;
   using Tester<T>::TestErrorCodes;
   using Tester<T>::GetExampleScalars;
+  using Tester<T>::GetOffsets;
 
   // Test settings for the regular test. Append to this list in case more tests are required.
   const std::vector<size_t> kVectorDims = { 7, 93, 4096 };
-  const std::vector<size_t> kOffsets = { 0, 10 };
+  const std::vector<size_t> kOffsets = GetOffsets();
   const std::vector<size_t> kIncrements = { 1, 2 };
   const std::vector<T> kAlphaValues = GetExampleScalars();
 

From 0f486d9b7442a314fd45f993137e6794372de1f5 Mon Sep 17 00:00:00 2001
From: CNugteren <web@cedricnugteren.nl>
Date: Sat, 20 Jun 2015 14:13:54 +0200
Subject: [PATCH 4/6] Automatically skips tests with unsupported precision

---
 include/internal/routine.h   |  4 ----
 include/internal/utilities.h |  4 ++++
 test/correctness/testabc.cc  |  2 ++
 test/correctness/testabc.h   |  1 +
 test/correctness/testaxy.cc  |  2 ++
 test/correctness/testaxy.h   |  1 +
 test/correctness/tester.cc   | 25 ++++++++++++++++++++++++-
 test/correctness/tester.h    |  3 +++
 test/correctness/testxy.cc   |  2 ++
 test/correctness/testxy.h    |  3 ++-
 10 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/include/internal/routine.h b/include/internal/routine.h
index 1b2e0dbb..a65ced20 100644
--- a/include/internal/routine.h
+++ b/include/internal/routine.h
@@ -29,10 +29,6 @@ namespace clblast {
 class Routine {
  public:
 
-  // Khronos OpenCL extensions
-  const std::string kKhronosHalfPrecision = "cl_khr_fp16";
-  const std::string kKhronosDoublePrecision = "cl_khr_fp64";
-
   // The cache of compiled OpenCL programs, along with some meta-data
   struct ProgramCache {
     Program program;
diff --git a/include/internal/utilities.h b/include/internal/utilities.h
index 4cefdc4a..6ad17a6a 100644
--- a/include/internal/utilities.h
+++ b/include/internal/utilities.h
@@ -31,6 +31,10 @@ namespace clblast {
 using float2 = std::complex<float>;
 using double2 = std::complex<double>;
 
+// Khronos OpenCL extensions
+const std::string kKhronosHalfPrecision = "cl_khr_fp16";
+const std::string kKhronosDoublePrecision = "cl_khr_fp64";
+
 // =================================================================================================
 
 // The routine-specific arguments in string form
diff --git a/test/correctness/testabc.cc b/test/correctness/testabc.cc
index 107ffb70..eed17560 100644
--- a/test/correctness/testabc.cc
+++ b/test/correctness/testabc.cc
@@ -46,6 +46,7 @@ TestABC<T>::TestABC(int argc, char *argv[], const bool silent,
 // Tests the routine for a wide variety of parameters
 template <typename T>
 void TestABC<T>::TestRegular(Arguments<T> &args, const std::string &name) {
+  if (!PrecisionSupported()) { return; }
   TestStart("regular behaviour", name);
 
   // Computes whether or not the matrices are transposed. Note that we assume a default of
@@ -161,6 +162,7 @@ void TestABC<T>::TestRegular(Arguments<T> &args, const std::string &name) {
 // does not test for results (if any).
 template <typename T>
 void TestABC<T>::TestInvalidBufferSizes(Arguments<T> &args, const std::string &name) {
+  if (!PrecisionSupported()) { return; }
   TestStart("invalid buffer sizes", name);
 
   // Sets example test parameters
diff --git a/test/correctness/testabc.h b/test/correctness/testabc.h
index b7601024..2c44d532 100644
--- a/test/correctness/testabc.h
+++ b/test/correctness/testabc.h
@@ -42,6 +42,7 @@ class TestABC: public Tester<T> {
   using Tester<T>::TestErrorCodes;
   using Tester<T>::GetExampleScalars;
   using Tester<T>::GetOffsets;
+  using Tester<T>::PrecisionSupported;
 
   // Test settings for the regular test. Append to this list in case more tests are required.
   const std::vector<size_t> kMatrixDims = { 7, 64 };
diff --git a/test/correctness/testaxy.cc b/test/correctness/testaxy.cc
index 3588573a..cb5e9923 100644
--- a/test/correctness/testaxy.cc
+++ b/test/correctness/testaxy.cc
@@ -47,6 +47,7 @@ TestAXY<T>::TestAXY(int argc, char *argv[], const bool silent,
 // Tests the routine for a wide variety of parameters
 template <typename T>
 void TestAXY<T>::TestRegular(Arguments<T> &args, const std::string &name) {
+  if (!PrecisionSupported()) { return; }
   TestStart("regular behaviour", name);
 
   // Iterates over the dimension for the matrix and vectors
@@ -151,6 +152,7 @@ void TestAXY<T>::TestRegular(Arguments<T> &args, const std::string &name) {
 // does not test for results (if any).
 template <typename T>
 void TestAXY<T>::TestInvalidBufferSizes(Arguments<T> &args, const std::string &name) {
+  if (!PrecisionSupported()) { return; }
   TestStart("invalid buffer sizes", name);
 
   // Sets example test parameters
diff --git a/test/correctness/testaxy.h b/test/correctness/testaxy.h
index 00d4f6ec..fa2c4a98 100644
--- a/test/correctness/testaxy.h
+++ b/test/correctness/testaxy.h
@@ -42,6 +42,7 @@ class TestAXY: public Tester<T> {
   using Tester<T>::TestErrorCodes;
   using Tester<T>::GetExampleScalars;
   using Tester<T>::GetOffsets;
+  using Tester<T>::PrecisionSupported;
 
   // Test settings for the regular test. Append to this list in case more tests are required.
   const std::vector<size_t> kMatrixVectorDims = { 61, 512 };
diff --git a/test/correctness/tester.cc b/test/correctness/tester.cc
index 09b0b5bd..74b6679d 100644
--- a/test/correctness/tester.cc
+++ b/test/correctness/tester.cc
@@ -56,8 +56,18 @@ Tester<T>::Tester(int argc, char *argv[], const bool silent,
 
   // Prints the header
   fprintf(stdout, "* Running on OpenCL device '%s'.\n", device_.Name().c_str());
-  fprintf(stdout, "* Starting tests for the %s'%s'%s routine. Legend:\n",
+  fprintf(stdout, "* Starting tests for the %s'%s'%s routine.",
           kPrintMessage.c_str(), name.c_str(), kPrintEnd.c_str());
+
+  // Checks whether the precision is supported
+  if (!PrecisionSupported()) {
+    fprintf(stdout, "\n* Tests skipped: %sUnsupported precision%s\n",
+            kPrintWarning.c_str(), kPrintEnd.c_str());
+    return;
+  }
+
+  // Prints the legend
+  fprintf(stdout, " Legend:\n");
   fprintf(stdout, "   %s -> Test produced correct results\n", kSuccessData.c_str());
   fprintf(stdout, "   %s -> Test returned the correct error code\n", kSuccessStatus.c_str());
   fprintf(stdout, "   %s -> Test produced incorrect results\n", kErrorData.c_str());
@@ -290,6 +300,19 @@ const std::vector<size_t> Tester<T>::GetOffsets() {
 
 // =================================================================================================
 
+template <> bool Tester<float>::PrecisionSupported() const { return true; }
+template <> bool Tester<float2>::PrecisionSupported() const { return true; }
+template <> bool Tester<double>::PrecisionSupported() const {
+  auto extensions = device_.Extensions();
+  return (extensions.find(kKhronosDoublePrecision) == std::string::npos) ? false : true;
+}
+template <> bool Tester<double2>::PrecisionSupported() const {
+  auto extensions = device_.Extensions();
+  return (extensions.find(kKhronosDoublePrecision) == std::string::npos) ? false : true;
+}
+
+// =================================================================================================
+
 // A test can either pass, be skipped, or fail
 template <typename T>
 void Tester<T>::ReportPass() {
diff --git a/test/correctness/tester.h b/test/correctness/tester.h
index 1085c472..b627d8f1 100644
--- a/test/correctness/tester.h
+++ b/test/correctness/tester.h
@@ -100,6 +100,9 @@ class Tester {
   // Retrieves a list of offset values to test
   const std::vector<size_t> GetOffsets();
 
+  // Returns false is this precision is not supported by the device
+  bool PrecisionSupported() const;
+
   // The help-message
   std::string help_;
 
diff --git a/test/correctness/testxy.cc b/test/correctness/testxy.cc
index b2d6c147..5c86182c 100644
--- a/test/correctness/testxy.cc
+++ b/test/correctness/testxy.cc
@@ -44,6 +44,7 @@ TestXY<T>::TestXY(int argc, char *argv[], const bool silent,
 // Tests the routine for a wide variety of parameters
 template <typename T>
 void TestXY<T>::TestRegular(Arguments<T> &args, const std::string &name) {
+  if (!PrecisionSupported()) { return; }
   TestStart("regular behaviour", name);
 
   // Iterates over the vector dimension
@@ -122,6 +123,7 @@ void TestXY<T>::TestRegular(Arguments<T> &args, const std::string &name) {
 // does not test for results (if any).
 template <typename T>
 void TestXY<T>::TestInvalidBufferSizes(Arguments<T> &args, const std::string &name) {
+  if (!PrecisionSupported()) { return; }
   TestStart("invalid buffer sizes", name);
 
   // Sets example test parameters
diff --git a/test/correctness/testxy.h b/test/correctness/testxy.h
index 8f2b6876..ec2cbcc7 100644
--- a/test/correctness/testxy.h
+++ b/test/correctness/testxy.h
@@ -40,11 +40,12 @@ class TestXY: public Tester<T> {
   using Tester<T>::TestErrorCodes;
   using Tester<T>::GetExampleScalars;
   using Tester<T>::GetOffsets;
+  using Tester<T>::PrecisionSupported;
 
   // Test settings for the regular test. Append to this list in case more tests are required.
   const std::vector<size_t> kVectorDims = { 7, 93, 4096 };
   const std::vector<size_t> kOffsets = GetOffsets();
-  const std::vector<size_t> kIncrements = { 1, 2 };
+  const std::vector<size_t> kIncrements = { 1, 2, 7 };
   const std::vector<T> kAlphaValues = GetExampleScalars();
 
   // Test settings for the invalid test

From c365614eb52c113f66f9a486aac8d11b8a28dab6 Mon Sep 17 00:00:00 2001
From: CNugteren <web@cedricnugteren.nl>
Date: Sat, 20 Jun 2015 16:43:50 +0200
Subject: [PATCH 5/6] More detailed test passed/skipped/failure reporting

---
 README.md                  |  1 -
 test/correctness/tester.cc | 38 ++++++++++++++++++++------------------
 test/correctness/tester.h  |  5 +++--
 3 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/README.md b/README.md
index 7c2b6208..1bed1146 100644
--- a/README.md
+++ b/README.md
@@ -225,5 +225,4 @@ To-do list before release of version 1.0
 - Further reduce the likelihood of crashes:
   * Add checks for proper command-line arguments in the tuner, tester and client
   * Add checks for valid database parameters
-  * Distinguish between short (smoke) and long tests
   * Test in multi-threaded environments
diff --git a/test/correctness/tester.cc b/test/correctness/tester.cc
index 74b6679d..501f1906 100644
--- a/test/correctness/tester.cc
+++ b/test/correctness/tester.cc
@@ -45,10 +45,11 @@ Tester<T>::Tester(int argc, char *argv[], const bool silent,
     error_log_{},
     num_passed_{0},
     num_skipped_{0},
-    num_errors_{0},
+    num_failed_{0},
     print_count_{0},
-    tests_failed_{0},
     tests_passed_{0},
+    tests_skipped_{0},
+    tests_failed_{0},
     options_{options} {
 
   // Prints the help message (command-line arguments)
@@ -61,7 +62,7 @@ Tester<T>::Tester(int argc, char *argv[], const bool silent,
 
   // Checks whether the precision is supported
   if (!PrecisionSupported()) {
-    fprintf(stdout, "\n* Tests skipped: %sUnsupported precision%s\n",
+    fprintf(stdout, "\n* All tests skipped: %sUnsupported precision%s\n",
             kPrintWarning.c_str(), kPrintEnd.c_str());
     return;
   }
@@ -87,14 +88,13 @@ Tester<T>::Tester(int argc, char *argv[], const bool silent,
 // Destructor prints the summary of the test cases and cleans-up the clBLAS library
 template <typename T>
 Tester<T>::~Tester() {
-  fprintf(stdout, "* Completed all test-cases for this routine. Results:\n");
-  fprintf(stdout, "   %lu test(s) succeeded\n", tests_passed_);
-  if (tests_failed_ != 0) {
-    fprintf(stdout, "   %s%lu test(s) failed%s\n",
-            kPrintError.c_str(), tests_failed_, kPrintEnd.c_str());
-  }
-  else {
-    fprintf(stdout, "   %lu test(s) failed\n", tests_failed_);
+  if (PrecisionSupported()) {
+    fprintf(stdout, "* Completed all test-cases for this routine. Results:\n");
+    fprintf(stdout, "   %lu test(s) passed\n", tests_passed_);
+    if (tests_skipped_ > 0) { fprintf(stdout, "%s", kPrintWarning.c_str()); }
+    fprintf(stdout, "   %lu test(s) skipped%s\n", tests_skipped_, kPrintEnd.c_str());
+    if (tests_failed_ > 0) { fprintf(stdout, "%s", kPrintError.c_str()); }
+    fprintf(stdout, "   %lu test(s) failed%s\n", tests_failed_, kPrintEnd.c_str());
   }
   fprintf(stdout, "\n");
   clblasTeardown();
@@ -117,7 +117,7 @@ void Tester<T>::TestStart(const std::string &test_name, const std::string &test_
   error_log_.clear();
   num_passed_ = 0;
   num_skipped_ = 0;
-  num_errors_ = 0;
+  num_failed_ = 0;
   print_count_ = 0;
 }
 
@@ -126,7 +126,9 @@ void Tester<T>::TestStart(const std::string &test_name, const std::string &test_
 template <typename T>
 void Tester<T>::TestEnd() {
   fprintf(stdout, "\n");
-  if (error_log_.size() == 0) { tests_passed_++; } else { tests_failed_++; }
+  tests_passed_ += num_passed_;
+  tests_failed_ += num_skipped_;
+  tests_failed_ += num_failed_;
 
   // Prints details of all error occurences for these tests
   for (auto &entry: error_log_) {
@@ -160,7 +162,7 @@ void Tester<T>::TestEnd() {
   }
 
   // Prints a test summary
-  auto pass_rate = 100*num_passed_ / static_cast<float>(num_passed_ + num_skipped_ + num_errors_);
+  auto pass_rate = 100*num_passed_ / static_cast<float>(num_passed_ + num_skipped_ + num_failed_);
   fprintf(stdout, "   Pass rate %s%5.1lf%%%s:", kPrintMessage.c_str(), pass_rate, kPrintEnd.c_str());
   fprintf(stdout, " %lu passed /", num_passed_);
   if (num_skipped_ != 0) {
@@ -169,11 +171,11 @@ void Tester<T>::TestEnd() {
   else {
     fprintf(stdout, " %lu skipped /", num_skipped_);
   }
-  if (num_errors_ != 0) {
-    fprintf(stdout, " %s%lu failed%s\n", kPrintError.c_str(), num_errors_, kPrintEnd.c_str());
+  if (num_failed_ != 0) {
+    fprintf(stdout, " %s%lu failed%s\n", kPrintError.c_str(), num_failed_, kPrintEnd.c_str());
   }
   else {
-    fprintf(stdout, " %lu failed\n", num_errors_);
+    fprintf(stdout, " %lu failed\n", num_failed_);
   }
 }
 
@@ -325,7 +327,7 @@ void Tester<T>::ReportSkipped() {
 template <typename T>
 void Tester<T>::ReportError(const ErrorLogEntry &error_log_entry) {
   error_log_.push_back(error_log_entry);
-  num_errors_++;
+  num_failed_++;
 }
 
 // =================================================================================================
diff --git a/test/correctness/tester.h b/test/correctness/tester.h
index b627d8f1..3b6fa059 100644
--- a/test/correctness/tester.h
+++ b/test/correctness/tester.h
@@ -129,14 +129,15 @@ class Tester {
   std::vector<ErrorLogEntry> error_log_;
   size_t num_passed_;
   size_t num_skipped_;
-  size_t num_errors_;
+  size_t num_failed_;
 
   // Counting the amount of errors printed on this row
   size_t print_count_;
 
   // Counting the number of test-cases with and without failures
-  size_t tests_failed_;
   size_t tests_passed_;
+  size_t tests_skipped_;
+  size_t tests_failed_;
 
   // Arguments relevant for a specific routine
   std::vector<std::string> options_;

From 84dd6ba1d767ffa576b31c93812bf38babb4eef3 Mon Sep 17 00:00:00 2001
From: CNugteren <web@cedricnugteren.nl>
Date: Sat, 20 Jun 2015 16:47:50 +0200
Subject: [PATCH 6/6] Updated changelog with testing improvements

---
 CHANGELOG | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 7a8bfb54..6d8635a4 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,17 +1,18 @@
 
 Development version (next release)
 - Added support for complex conjugate transpose
-- Some host-code performance improvements
+- Several host-code performance improvements
+- Improved testing infrastructure and coverage
 - Added level-2 routines:
-    SGEMV/DGEMV/CGEMV/ZGEMV
+  * SGEMV/DGEMV/CGEMV/ZGEMV
 - Added level-3 routines:
-    CGEMM/ZGEMM
-    CSYMM/ZSYMM
+  * CGEMM/ZGEMM
+  * CSYMM/ZSYMM
 
 Version 0.1.0
 - Initial preview version release to GitHub
 - Supported level-1 routines:
-    SAXPY/DAXPY/CAXPY/ZAXPY
+  * SAXPY/DAXPY/CAXPY/ZAXPY
 - Supported level-3 routines:
-    SGEMM/DGEMM
-    SSYMM/DSYMM
+  * SGEMM/DGEMM
+  * SSYMM/DSYMM