Add Catch.hpp for unit testing, with appveyor hooks

docs/credits_acknowledgements.rst

@@ -96,6 +96,10 @@ The following libraries and components are incorporated into RenderDoc, listed h
 
   Provides the ability to disassemble shaders from any API representation into compiled GCN ISA for lower level analysis.
 
+* `Catch <https://github.com/philsquared/Catch>`_ - Copyright (c) 2012 Two Blue Cubes Ltd., distributed under the Boost Software License.
+
+  Implements unit testing during development.
+
 Thanks
 ------
 

renderdoc/3rdparty/catch/LICENSE.txt

@@ -0,0 +1,23 @@
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

renderdoc/3rdparty/catch/catch.cpp

@@ -0,0 +1,309 @@
+/******************************************************************************
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2017 Baldur Karlsson
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ ******************************************************************************/
+
+#define CATCH_CONFIG_RUNNER
+#define CATCH_CONFIG_NOSTDOUT
+#include "catch.hpp"
+#include "api/replay/renderdoc_replay.h"
+#include "serialise/serialiser.h"
+#include "serialise/string_utils.h"
+
+struct AppVeyorListener : Catch::TestEventListenerBase
+{
+  using TestEventListenerBase::TestEventListenerBase;    // inherit constructor
+
+  bool enabled = false;
+  std::string hostname;
+  uint16_t port = 0;
+
+  virtual void testRunStarting(Catch::TestRunInfo const &testRunInfo)
+  {
+    const char *url = Process::GetEnvVariable("APPVEYOR_API_URL");
+
+    if(url)
+    {
+      if(strncmp(url, "http://", 7))
+        return;
+
+      url += 7;
+
+      const char *sep = strchr(url, ':');
+
+      if(!sep)
+        return;
+
+      hostname = std::string(url, sep);
+
+      url = sep + 1;
+
+      port = 0;
+      while(*url >= '0' && *url <= '9')
+      {
+        port *= 10;
+        port += int((*url) - '0');
+        url++;
+      }
+
+      Network::Socket *sock = Network::CreateClientSocket(hostname.c_str(), port, 10);
+
+      if(sock)
+        enabled = true;
+
+      SAFE_DELETE(sock);
+    }
+  }
+
+  std::string curTest;
+  std::vector<std::string> sectionStack;
+
+  virtual void testCaseStarting(Catch::TestCaseInfo const &testInfo) { curTest = testInfo.name; }
+  virtual void sectionStarting(Catch::SectionInfo const &sectionInfo)
+  {
+    if(curTest == sectionInfo.name)
+      return;
+
+    sectionStack.push_back(sectionInfo.name);
+
+    if(enabled)
+    {
+      Network::Socket *sock = Network::CreateClientSocket(hostname.c_str(), port, 10);
+
+      if(sock)
+      {
+        std::string req = MakeHTTPRequest();
+        sock->SendDataBlocking(req.c_str(), (uint32_t)req.size());
+      }
+
+      SAFE_DELETE(sock);
+    }
+  }
+
+  std::string errorList;
+
+  virtual bool assertionEnded(Catch::AssertionStats const &assertionStats)
+  {
+    using namespace Catch;
+
+    if(!assertionStats.assertionResult.isOk())
+    {
+      std::ostringstream msg;
+      msg << assertionStats.assertionResult.getSourceInfo() << ": ";
+
+      switch(assertionStats.assertionResult.getResultType())
+      {
+        case ResultWas::ExpressionFailed: msg << "Failed"; break;
+        case ResultWas::ThrewException: msg << "Threw exception"; break;
+        case ResultWas::FatalErrorCondition: msg << "Fatal error'd"; break;
+        case ResultWas::DidntThrowException: msg << "Didn't throw expected exception"; break;
+        case ResultWas::ExplicitFailure: msg << "Explicitly failed"; break;
+
+        case ResultWas::Ok:
+        case ResultWas::Info:
+        case ResultWas::Warning:
+        case ResultWas::Unknown:
+        case ResultWas::FailureBit:
+        case ResultWas::Exception: break;
+      }
+
+      if(assertionStats.infoMessages.size() >= 1)
+        msg << " with message(s):";
+      for(auto it = assertionStats.infoMessages.begin(); it != assertionStats.infoMessages.end(); ++it)
+        msg << "\n" << it->message;
+
+      if(assertionStats.assertionResult.hasExpression())
+      {
+        msg << "\n  " << assertionStats.assertionResult.getExpressionInMacro()
+            << "\nwith expansion:\n  " << assertionStats.assertionResult.getExpandedExpression()
+            << "\n";
+      }
+
+      errorList += msg.str();
+    }
+
+    return true;
+  }
+
+  virtual void sectionEnded(Catch::SectionStats const &sectionStats)
+  {
+    if(curTest == sectionStats.sectionInfo.name)
+      return;
+
+    if(enabled)
+    {
+      Network::Socket *sock = Network::CreateClientSocket(hostname.c_str(), port, 10);
+
+      if(sock)
+      {
+        std::string req = MakeHTTPRequest(sectionStats.durationInSeconds * 1000.0,
+                                          sectionStats.assertions.allOk());
+        sock->SendDataBlocking(req.c_str(), (uint32_t)req.size());
+      }
+
+      errorList.clear();
+
+      SAFE_DELETE(sock);
+    }
+
+    sectionStack.pop_back();
+  }
+
+private:
+  std::string MakeHTTPRequest(double msDuration = -1.0, bool passed = false)
+  {
+    std::string json;
+
+    bool update = msDuration >= 0.0;
+
+    const char *outcome = "Running";
+
+    if(update)
+      outcome = passed ? "Passed" : "Failed";
+
+    std::string testName;
+    for(const std::string &section : sectionStack)
+    {
+      if(!testName.empty())
+        testName += " > ";
+      testName += section;
+    }
+
+    json = StringFormat::Fmt(R"(
+{
+    "testName": "%s",
+    "testFramework": "Catch.hpp",
+    "fileName": "%s",
+    "outcome": "%s",
+    "durationMilliseconds": "%.0f",
+    "ErrorMessage": "%s",
+    "ErrorStackTrace": "",
+    "StdOut": "",
+    "StdErr": ""
+})",
+                             testName.c_str(), curTest.c_str(), outcome,
+                             RDCMAX(msDuration * 1000.0, 0.0), escape(trim(errorList)).c_str());
+
+    std::string http;
+    http += StringFormat::Fmt("%s /api/tests HTTP/1.1\r\n", update ? "PUT" : "POST");
+    http += StringFormat::Fmt("Host: %s\r\n", hostname.c_str());
+    http += "Connection: close\r\n";
+    http += "Content-Type: application/json\r\n";
+    http += StringFormat::Fmt("Content-Length: %zu\r\n", json.size());
+    http += "User-Agent: Catch.hpp appveyor updater\r\n";
+    http += "\r\n";
+    return http + json;
+  }
+
+  std::string escape(const std::string &input)
+  {
+    std::string ret = input;
+    size_t i = ret.find_first_of("\"\n\\", 0);
+    while(i != std::string::npos)
+    {
+      if(ret[i] == '"')
+        ret.replace(i, 1, "\\\"");
+      else if(ret[i] == '\\')
+        ret.replace(i, 1, "\\\\");
+      else if(ret[i] == '\n')
+        ret.replace(i, 1, "\\n");
+
+      i = ret.find_first_of("\"\n\\", i + 2);
+    }
+
+    return ret;
+  }
+};
+CATCH_REGISTER_LISTENER(AppVeyorListener)
+
+class LogOutputter : public std::stringbuf
+{
+public:
+  LogOutputter() {}
+  virtual int sync() override
+  {
+    std::string msg = this->str();
+    OSUtility::WriteOutput(OSUtility::Output_DebugMon, msg.c_str());
+    OSUtility::WriteOutput(OSUtility::Output_StdOut, msg.c_str());
+    this->str("");
+    return 0;
+  }
+
+  // force a sync on every output
+  virtual std::streamsize xsputn(const char *s, std::streamsize n) override
+  {
+    std::streamsize ret = std::stringbuf::xsputn(s, n);
+    sync();
+    return ret;
+  }
+};
+
+std::ostream *stream = NULL;
+
+namespace Catch
+{
+std::ostream &cout()
+{
+  return *stream;
+}
+std::ostream &cerr()
+{
+  return *stream;
+}
+std::ostream &clog()
+{
+  return *stream;
+}
+}
+
+extern "C" RENDERDOC_API int RENDERDOC_CC
+RENDERDOC_RunUnitTests(const rdctype::str &command, const rdctype::array<rdctype::str> &args)
+{
+  LogOutputter logbuf;
+  std::ostream logstream(&logbuf);
+  stream = &logstream;
+
+  Catch::Session session;
+
+  session.configData().name = "RenderDoc";
+  session.configData().shouldDebugBreak = OSUtility::DebuggerPresent();
+
+  const char **argv = new const char *[args.count + 1];
+  argv[0] = command.c_str();
+  for(int i = 0; i < args.count; i++)
+    argv[i + 1] = args.elems[i].c_str();
+
+  int ret = session.applyCommandLine(args.count + 1, argv);
+
+  delete[] argv;
+
+  // command line error
+  if(ret != 0)
+    return ret;
+
+  int numFailed = session.run();
+
+  // Note that on unices only the lower 8 bits are usually used, clamping
+  // the return value to 255 prevents false negative when some multiple
+  // of 256 tests has failed
+  return (numFailed < 0xff ? numFailed : 0xff);
+}

renderdoc/CMakeLists.txt

@@ -130,6 +130,8 @@ set(sources
     3rdparty/jpeg-compressor/jpgd.h
     3rdparty/jpeg-compressor/jpge.cpp
     3rdparty/jpeg-compressor/jpge.h
+    3rdparty/catch/catch.cpp
+    3rdparty/catch/catch.hpp
     3rdparty/lz4/lz4.c
     3rdparty/lz4/lz4.h
     3rdparty/stb/stb_image.h

renderdoc/api/replay/renderdoc_replay.h

@@ -1607,3 +1607,7 @@ DOCUMENT("Internal function that attempts to modify APK contents, adding Vulkan
 extern "C" RENDERDOC_API bool RENDERDOC_CC RENDERDOC_AddLayerToAndroidPackage(const char *host,
                                                                               const char *exe,
                                                                               float *progress);
+
+DOCUMENT("Internal function that runs unit tests.");
+extern "C" RENDERDOC_API int RENDERDOC_CC
+RENDERDOC_RunUnitTests(const rdctype::str &command, const rdctype::array<rdctype::str> &args);

renderdoc/common/globalconfig.h

@@ -171,3 +171,5 @@ enum
 #define FORCE_DEBUG_LOGS OPTION_OFF
 // this strips them completely
 #define STRIP_DEBUG_LOGS OPTION_OFF
+
+#define ENABLE_UNIT_TESTS RDOC_DEVEL

renderdoc/os/os_specific.h

@@ -52,6 +52,8 @@ void RegisterEnvironmentModification(EnvironmentModification modif);
 
 void ApplyEnvironmentModification();
 
+const char *GetEnvVariable(const char *name);
+
 bool CanGlobalHook();
 bool StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOptions &opts);
 bool IsGlobalHookActive();

renderdoc/os/posix/posix_process.cpp

@@ -212,6 +212,11 @@ void Process::ApplyEnvironmentModification()
   modifications.clear();
 }
 
+const char *Process::GetEnvVariable(const char *name)
+{
+  return getenv(name);
+}
+
 static pid_t RunProcess(const char *app, const char *workingDir, const char *cmdLine, char **envp,
                         int stdoutPipe[2] = NULL, int stderrPipe[2] = NULL)
 {

renderdoc/os/win32/win32_process.cpp

@@ -153,6 +153,14 @@ void Process::ApplyEnvironmentModification()
   modifications.clear();
 }
 
+const char *Process::GetEnvVariable(const char *name)
+{
+  static char buf[1024] = {};
+  size_t reqSize = 1024;
+  getenv_s(&reqSize, buf, name);
+  return buf;
+}
+
 // helpers for various shims and dlls etc, not part of the public API
 extern "C" __declspec(dllexport) void __cdecl INTERNAL_GetTargetControlIdent(uint32_t *ident)
 {

renderdoc/os/win32/win32_stringio.cpp

@@ -723,9 +723,9 @@ void WriteOutput(int channel, const char *str)
   if(channel == OSUtility::Output_DebugMon)
     OutputDebugStringW(wstr.c_str());
   else if(channel == OSUtility::Output_StdOut)
-    fwprintf(stdout, L"%ls", wstr.c_str());
+    WriteConsoleW(GetStdHandle(STD_OUTPUT_HANDLE), wstr.c_str(), DWORD(wstr.size()), NULL, NULL);
   else if(channel == OSUtility::Output_StdErr)
-    fwprintf(stderr, L"%ls", wstr.c_str());
+    WriteConsoleW(GetStdHandle(STD_ERROR_HANDLE), wstr.c_str(), DWORD(wstr.size()), NULL, NULL);
 }
 
 uint64_t GetMachineIdent()

renderdoc/renderdoc.vcxproj

@@ -104,6 +104,7 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
+    <ClInclude Include="3rdparty\catch\catch.hpp" />
     <ClInclude Include="3rdparty\jpeg-compressor\jpgd.h" />
     <ClInclude Include="3rdparty\jpeg-compressor\jpge.h" />
     <ClInclude Include="3rdparty\lz4\lz4.h" />
@@ -175,6 +176,7 @@
     <ClInclude Include="serialise\string_utils.h" />
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="3rdparty\catch\catch.cpp" />
     <ClCompile Include="3rdparty\jpeg-compressor\jpgd.cpp">
       <PrecompiledHeader>NotUsing</PrecompiledHeader>
       <ForcedIncludeFiles>

renderdoc/renderdoc.vcxproj.filters

@@ -88,6 +88,9 @@
     <Filter Include="PCH">
       <UniqueIdentifier>{364a44c3-ab9f-44d1-ab3f-e9f640202224}</UniqueIdentifier>
     </Filter>
+    <Filter Include="3rdparty\catch">
+      <UniqueIdentifier>{af7852be-f538-40f0-be6a-19078ba71639}</UniqueIdentifier>
+    </Filter>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="maths\camera.h">
@@ -285,6 +288,9 @@
     <ClInclude Include="core\plugins.h">
       <Filter>Core</Filter>
     </ClInclude>
+    <ClInclude Include="3rdparty\catch\catch.hpp">
+      <Filter>3rdparty\catch</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="maths\camera.cpp">
@@ -477,6 +483,9 @@
     <ClCompile Include="core\plugins.cpp">
       <Filter>Core</Filter>
     </ClCompile>
+    <ClCompile Include="3rdparty\catch\catch.cpp">
+      <Filter>3rdparty\catch</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="os\win32\comexport.def">

renderdoc/replay/entry_points.cpp

@@ -23,6 +23,7 @@
  * THE SOFTWARE.
  ******************************************************************************/
 
+#include "3rdparty/catch/catch.hpp"
 #include "api/replay/renderdoc_replay.h"
 #include "api/replay/version.h"
 #include "common/common.h"
@@ -614,4 +615,4 @@ extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_ResourceFormatName(const Re
                                                                         rdctype::str &name)
 {
   name = ResourceFormatName(fmt);
-}
+}

renderdoc/serialise/string_utils.cpp

@@ -27,6 +27,7 @@
 #include <stdint.h>
 #include <wctype.h>
 #include <algorithm>
+#include "common/globalconfig.h"
 
 uint32_t strhash(const char *str, uint32_t seed)
 {

renderdoccmd/renderdoccmd.cpp

@@ -599,6 +599,30 @@ struct ReplayCommand : public Command
   }
 };
 
+struct TestCommand : public Command
+{
+  TestCommand(const GlobalEnvironment &env) : Command(env) {}
+  virtual void AddOptions(cmdline::parser &parser)
+  {
+    parser.add<string>("type", 't', "The type of test to run.", true, "",
+                       cmdline::oneof<string>("unit"));
+    parser.add("help", '\0', "print this message");
+    parser.stop_at_rest(true);
+  }
+  virtual const char *Description() { return "Run internal tests such as unit tests."; }
+  virtual bool IsInternalOnly() { return false; }
+  virtual bool IsCaptureCommand() { return false; }
+  virtual int Execute(cmdline::parser &parser, const CaptureOptions &)
+  {
+    std::vector<std::string> rest = parser.rest();
+
+    if(parser.get<string>("type") == "unit")
+      return RENDERDOC_RunUnitTests("renderdoccmd test --type unit", convertArgs(rest));
+
+    return 1;
+  }
+};
+
 struct CapAltBitCommand : public Command
 {
   CapAltBitCommand(const GlobalEnvironment &env) : Command(env) {}
@@ -736,6 +760,7 @@ int renderdoccmd(const GlobalEnvironment &env, std::vector<std::string> &argv)
     add_command("remoteserver", new RemoteServerCommand(env));
     add_command("replay", new ReplayCommand(env));
     add_command("capaltbit", new CapAltBitCommand(env));
+    add_command("test", new TestCommand(env));
 
     if(argv.size() <= 1)
     {