From 7ad31979821874b3e0768209122a934ec3448c01 Mon Sep 17 00:00:00 2001 From: baldurk Date: Wed, 9 Jan 2019 10:01:43 +0000 Subject: [PATCH] Revamp catch appveyor reporter to report all test results in a batch * We also only report one test per test-case, as trying to handle sections separately is unreliable. --- renderdoc/3rdparty/catch/catch.cpp | 252 ++++++++++++++--------------- 1 file changed, 124 insertions(+), 128 deletions(-) diff --git a/renderdoc/3rdparty/catch/catch.cpp b/renderdoc/3rdparty/catch/catch.cpp index 4ace07012..a16f308f3 100644 --- a/renderdoc/3rdparty/catch/catch.cpp +++ b/renderdoc/3rdparty/catch/catch.cpp @@ -41,68 +41,8 @@ struct AppVeyorListener : Catch::TestEventListenerBase { using TestEventListenerBase::TestEventListenerBase; // inherit constructor - bool enabled = false; - std::string hostname; - uint16_t port = 0; - - virtual void testRunStarting(Catch::TestRunInfo const &testRunInfo) - { - const char *url = Process::GetEnvVariable("APPVEYOR_API_URL"); - - if(url) - { - if(strncmp(url, "http://", 7)) - return; - - url += 7; - - const char *sep = strchr(url, ':'); - - if(!sep) - return; - - hostname = std::string(url, sep); - - url = sep + 1; - - port = 0; - while(*url >= '0' && *url <= '9') - { - port *= 10; - port += int((*url) - '0'); - url++; - } - - Network::Socket *sock = Network::CreateClientSocket(hostname.c_str(), port, 10); - - if(sock) - enabled = true; - - SAFE_DELETE(sock); - } - - Catch::TestEventListenerBase::testRunStarting(testRunInfo); - } - - virtual void sectionStarting(Catch::SectionInfo const §ionInfo) - { - Catch::TestEventListenerBase::sectionStarting(sectionInfo); - - if(enabled) - { - Network::Socket *sock = Network::CreateClientSocket(hostname.c_str(), port, 10); - - if(sock) - { - std::string req = MakeHTTPRequest(); - sock->SendDataBlocking(req.c_str(), (uint32_t)req.size()); - } - - SAFE_DELETE(sock); - } - } - std::string errorList; + double durationInSeconds = 0.0; virtual bool assertionEnded(Catch::AssertionStats const &assertionStats) { @@ -113,6 +53,15 @@ struct AppVeyorListener : Catch::TestEventListenerBase if(!assertionStats.assertionResult.isOk()) { std::ostringstream msg; + msg << "-------------------------------------------------------------------------------\n"; + for(size_t i = 0; i < m_sectionStack.size(); i++) + { + if(i > 0) + msg << " > "; + msg << m_sectionStack[i].name; + msg << "\n"; + } + msg << "-------------------------------------------------------------------------------\n"; msg << assertionStats.assertionResult.getSourceInfo() << ": "; switch(assertionStats.assertionResult.getResultType()) @@ -149,98 +98,145 @@ struct AppVeyorListener : Catch::TestEventListenerBase return true; } + struct TestCase + { + double durationInSeconds; + bool passed; + std::string errorList; + std::string name; + std::string filename; + + std::string MakeJSON(); + }; + + std::vector m_testcases; + virtual void sectionEnded(Catch::SectionStats const §ionStats) { - if(enabled) - { - Network::Socket *sock = Network::CreateClientSocket(hostname.c_str(), port, 10); - - if(sock) - { - std::string req = MakeHTTPRequest(sectionStats.durationInSeconds * 1000.0, - sectionStats.assertions.allOk()); - sock->SendDataBlocking(req.c_str(), (uint32_t)req.size()); - } - - errorList.clear(); - - SAFE_DELETE(sock); - } + durationInSeconds += sectionStats.durationInSeconds; Catch::TestEventListenerBase::sectionEnded(sectionStats); } -private: - std::string MakeHTTPRequest(double msDuration = -1.0, bool passed = false) + virtual void testCaseEnded(Catch::TestCaseStats const &testCaseStats) { - std::string json; + m_testcases.push_back({ + durationInSeconds, testCaseStats.totals.assertions.allOk(), errorList, + testCaseStats.testInfo.name, testCaseStats.testInfo.lineInfo.file, + }); - bool update = msDuration >= 0.0; + errorList.clear(); + durationInSeconds = 0.0; - const char *outcome = "Running"; + Catch::TestEventListenerBase::testCaseEnded(testCaseStats); + } - if(update) - outcome = passed ? "Passed" : "Failed"; + // we dump all the test data at the end, because appveyor can't be trusted to get it right + // incrementally. This means if the program crashes mid-run we don't get partial test output, but + // it should at least by identified as an issue. + virtual void testRunEnded(Catch::TestRunStats const &testRunStats) override + { + const char *url = Process::GetEnvVariable("APPVEYOR_API_URL"); - std::string fileName; - std::string testName; - for(const Catch::SectionInfo §ion : m_sectionStack) + if(url) { - if(!testName.empty()) - testName += " > "; - testName += section.name; + if(strncmp(url, "http://", 7)) + return; + + url += 7; + + const char *sep = strchr(url, ':'); + + if(!sep) + return; + + std::string hostname = std::string(url, sep); + + url = sep + 1; + + uint16_t port = 0; + while(*url >= '0' && *url <= '9') + { + port *= 10; + port += int((*url) - '0'); + url++; + } + + Network::Socket *sock = Network::CreateClientSocket(hostname.c_str(), port, 10); + + if(sock) + { + std::string json; + + json += "[\n"; + for(size_t i = 0; i < m_testcases.size(); i++) + { + json += m_testcases[i].MakeJSON(); + + if(i + 1 < m_testcases.size()) + json += ","; + + json += "\n"; + } + json += "]"; + + std::string http; + http += StringFormat::Fmt("POST /api/tests/batch HTTP/1.1\r\n"); + http += StringFormat::Fmt("Host: %s\r\n", hostname.c_str()); + http += "Connection: close\r\n"; + http += "Content-Type: application/json\r\n"; + http += StringFormat::Fmt("Content-Length: %zu\r\n", json.size()); + http += "User-Agent: Catch.hpp appveyor updater\r\n"; + http += "\r\n"; + http += json; + + sock->SendDataBlocking(http.c_str(), (uint32_t)http.size()); + } + + SAFE_DELETE(sock); } + } +}; - if(m_sectionStack.empty()) - fileName = currentTestCaseInfo->name; - else - fileName = m_sectionStack[0].lineInfo.file; - - json = StringFormat::Fmt(R"( +static std::string escape(const std::string &input) { + std::string ret = input; + size_t i = ret.find_first_of("\"\n\\", 0); + while(i != std::string::npos) + { + if(ret[i] == '"') + ret.replace(i, 1, "\\\""); + else if(ret[i] == '\\') + ret.replace(i, 1, "\\\\"); + else if(ret[i] == '\n') + ret.replace(i, 1, "\\n"); + + i = ret.find_first_of("\"\n\\", i + 2); + } + + return ret; +} + +std::string AppVeyorListener::TestCase::MakeJSON() +{ + std::string json; + + return StringFormat::Fmt( + R"({ "testName": "%s", "testFramework": "Catch.hpp", "fileName": "%s", "outcome": "%s", - "durationMilliseconds": "%.0f", + "durationMilliseconds": "%d", "ErrorMessage": "%s", "ErrorStackTrace": "", "StdOut": "", "StdErr": "" })", - testName.c_str(), fileName.c_str(), outcome, RDCMAX(msDuration, 0.0), - escape(trim(errorList)).c_str()); + escape(name).c_str(), escape(filename).c_str(), passed ? "Passed" : "Failed", + (int)RDCMAX(durationInSeconds * 1000.0, 0.0), escape(trim(errorList)).c_str()); +} - std::string http; - http += StringFormat::Fmt("%s /api/tests HTTP/1.1\r\n", update ? "PUT" : "POST"); - http += StringFormat::Fmt("Host: %s\r\n", hostname.c_str()); - http += "Connection: close\r\n"; - http += "Content-Type: application/json\r\n"; - http += StringFormat::Fmt("Content-Length: %zu\r\n", json.size()); - http += "User-Agent: Catch.hpp appveyor updater\r\n"; - http += "\r\n"; - return http + json; - } - - std::string escape(const std::string &input) - { - std::string ret = input; - size_t i = ret.find_first_of("\"\n\\", 0); - while(i != std::string::npos) - { - if(ret[i] == '"') - ret.replace(i, 1, "\\\""); - else if(ret[i] == '\\') - ret.replace(i, 1, "\\\\"); - else if(ret[i] == '\n') - ret.replace(i, 1, "\\n"); - - i = ret.find_first_of("\"\n\\", i + 2); - } - - return ret; - } -}; CATCH_REGISTER_LISTENER(AppVeyorListener) class LogOutputter : public std::stringbuf