mirror of
https://github.com/baldurk/renderdoc.git
synced 2026-09-15 18:25:49 +00:00
Revamp catch appveyor reporter to report all test results in a batch
* We also only report one test per test-case, as trying to handle sections separately is unreliable.
This commit is contained in:
baldurk
Vendored
+124
-128
@@ -41,68 +41,8 @@ struct AppVeyorListener : Catch::TestEventListenerBase
|
||||
{
|
||||
using TestEventListenerBase::TestEventListenerBase; // inherit constructor
|
||||
|
||||
bool enabled = false;
|
||||
std::string hostname;
|
||||
uint16_t port = 0;
|
||||
|
||||
virtual void testRunStarting(Catch::TestRunInfo const &testRunInfo)
|
||||
{
|
||||
const char *url = Process::GetEnvVariable("APPVEYOR_API_URL");
|
||||
|
||||
if(url)
|
||||
{
|
||||
if(strncmp(url, "http://", 7))
|
||||
return;
|
||||
|
||||
url += 7;
|
||||
|
||||
const char *sep = strchr(url, ':');
|
||||
|
||||
if(!sep)
|
||||
return;
|
||||
|
||||
hostname = std::string(url, sep);
|
||||
|
||||
url = sep + 1;
|
||||
|
||||
port = 0;
|
||||
while(*url >= '0' && *url <= '9')
|
||||
{
|
||||
port *= 10;
|
||||
port += int((*url) - '0');
|
||||
url++;
|
||||
}
|
||||
|
||||
Network::Socket *sock = Network::CreateClientSocket(hostname.c_str(), port, 10);
|
||||
|
||||
if(sock)
|
||||
enabled = true;
|
||||
|
||||
SAFE_DELETE(sock);
|
||||
}
|
||||
|
||||
Catch::TestEventListenerBase::testRunStarting(testRunInfo);
|
||||
}
|
||||
|
||||
virtual void sectionStarting(Catch::SectionInfo const §ionInfo)
|
||||
{
|
||||
Catch::TestEventListenerBase::sectionStarting(sectionInfo);
|
||||
|
||||
if(enabled)
|
||||
{
|
||||
Network::Socket *sock = Network::CreateClientSocket(hostname.c_str(), port, 10);
|
||||
|
||||
if(sock)
|
||||
{
|
||||
std::string req = MakeHTTPRequest();
|
||||
sock->SendDataBlocking(req.c_str(), (uint32_t)req.size());
|
||||
}
|
||||
|
||||
SAFE_DELETE(sock);
|
||||
}
|
||||
}
|
||||
|
||||
std::string errorList;
|
||||
double durationInSeconds = 0.0;
|
||||
|
||||
virtual bool assertionEnded(Catch::AssertionStats const &assertionStats)
|
||||
{
|
||||
@@ -113,6 +53,15 @@ struct AppVeyorListener : Catch::TestEventListenerBase
|
||||
if(!assertionStats.assertionResult.isOk())
|
||||
{
|
||||
std::ostringstream msg;
|
||||
msg << "-------------------------------------------------------------------------------\n";
|
||||
for(size_t i = 0; i < m_sectionStack.size(); i++)
|
||||
{
|
||||
if(i > 0)
|
||||
msg << " > ";
|
||||
msg << m_sectionStack[i].name;
|
||||
msg << "\n";
|
||||
}
|
||||
msg << "-------------------------------------------------------------------------------\n";
|
||||
msg << assertionStats.assertionResult.getSourceInfo() << ": ";
|
||||
|
||||
switch(assertionStats.assertionResult.getResultType())
|
||||
@@ -149,98 +98,145 @@ struct AppVeyorListener : Catch::TestEventListenerBase
|
||||
return true;
|
||||
}
|
||||
|
||||
struct TestCase
|
||||
{
|
||||
double durationInSeconds;
|
||||
bool passed;
|
||||
std::string errorList;
|
||||
std::string name;
|
||||
std::string filename;
|
||||
|
||||
std::string MakeJSON();
|
||||
};
|
||||
|
||||
std::vector<TestCase> m_testcases;
|
||||
|
||||
virtual void sectionEnded(Catch::SectionStats const §ionStats)
|
||||
{
|
||||
if(enabled)
|
||||
{
|
||||
Network::Socket *sock = Network::CreateClientSocket(hostname.c_str(), port, 10);
|
||||
|
||||
if(sock)
|
||||
{
|
||||
std::string req = MakeHTTPRequest(sectionStats.durationInSeconds * 1000.0,
|
||||
sectionStats.assertions.allOk());
|
||||
sock->SendDataBlocking(req.c_str(), (uint32_t)req.size());
|
||||
}
|
||||
|
||||
errorList.clear();
|
||||
|
||||
SAFE_DELETE(sock);
|
||||
}
|
||||
durationInSeconds += sectionStats.durationInSeconds;
|
||||
|
||||
Catch::TestEventListenerBase::sectionEnded(sectionStats);
|
||||
}
|
||||
|
||||
private:
|
||||
std::string MakeHTTPRequest(double msDuration = -1.0, bool passed = false)
|
||||
virtual void testCaseEnded(Catch::TestCaseStats const &testCaseStats)
|
||||
{
|
||||
std::string json;
|
||||
m_testcases.push_back({
|
||||
durationInSeconds, testCaseStats.totals.assertions.allOk(), errorList,
|
||||
testCaseStats.testInfo.name, testCaseStats.testInfo.lineInfo.file,
|
||||
});
|
||||
|
||||
bool update = msDuration >= 0.0;
|
||||
errorList.clear();
|
||||
durationInSeconds = 0.0;
|
||||
|
||||
const char *outcome = "Running";
|
||||
Catch::TestEventListenerBase::testCaseEnded(testCaseStats);
|
||||
}
|
||||
|
||||
if(update)
|
||||
outcome = passed ? "Passed" : "Failed";
|
||||
// we dump all the test data at the end, because appveyor can't be trusted to get it right
|
||||
// incrementally. This means if the program crashes mid-run we don't get partial test output, but
|
||||
// it should at least by identified as an issue.
|
||||
virtual void testRunEnded(Catch::TestRunStats const &testRunStats) override
|
||||
{
|
||||
const char *url = Process::GetEnvVariable("APPVEYOR_API_URL");
|
||||
|
||||
std::string fileName;
|
||||
std::string testName;
|
||||
for(const Catch::SectionInfo §ion : m_sectionStack)
|
||||
if(url)
|
||||
{
|
||||
if(!testName.empty())
|
||||
testName += " > ";
|
||||
testName += section.name;
|
||||
if(strncmp(url, "http://", 7))
|
||||
return;
|
||||
|
||||
url += 7;
|
||||
|
||||
const char *sep = strchr(url, ':');
|
||||
|
||||
if(!sep)
|
||||
return;
|
||||
|
||||
std::string hostname = std::string(url, sep);
|
||||
|
||||
url = sep + 1;
|
||||
|
||||
uint16_t port = 0;
|
||||
while(*url >= '0' && *url <= '9')
|
||||
{
|
||||
port *= 10;
|
||||
port += int((*url) - '0');
|
||||
url++;
|
||||
}
|
||||
|
||||
Network::Socket *sock = Network::CreateClientSocket(hostname.c_str(), port, 10);
|
||||
|
||||
if(sock)
|
||||
{
|
||||
std::string json;
|
||||
|
||||
json += "[\n";
|
||||
for(size_t i = 0; i < m_testcases.size(); i++)
|
||||
{
|
||||
json += m_testcases[i].MakeJSON();
|
||||
|
||||
if(i + 1 < m_testcases.size())
|
||||
json += ",";
|
||||
|
||||
json += "\n";
|
||||
}
|
||||
json += "]";
|
||||
|
||||
std::string http;
|
||||
http += StringFormat::Fmt("POST /api/tests/batch HTTP/1.1\r\n");
|
||||
http += StringFormat::Fmt("Host: %s\r\n", hostname.c_str());
|
||||
http += "Connection: close\r\n";
|
||||
http += "Content-Type: application/json\r\n";
|
||||
http += StringFormat::Fmt("Content-Length: %zu\r\n", json.size());
|
||||
http += "User-Agent: Catch.hpp appveyor updater\r\n";
|
||||
http += "\r\n";
|
||||
http += json;
|
||||
|
||||
sock->SendDataBlocking(http.c_str(), (uint32_t)http.size());
|
||||
}
|
||||
|
||||
SAFE_DELETE(sock);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if(m_sectionStack.empty())
|
||||
fileName = currentTestCaseInfo->name;
|
||||
else
|
||||
fileName = m_sectionStack[0].lineInfo.file;
|
||||
|
||||
json = StringFormat::Fmt(R"(
|
||||
static std::string escape(const std::string &input)
|
||||
{
|
||||
std::string ret = input;
|
||||
size_t i = ret.find_first_of("\"\n\\", 0);
|
||||
while(i != std::string::npos)
|
||||
{
|
||||
if(ret[i] == '"')
|
||||
ret.replace(i, 1, "\\\"");
|
||||
else if(ret[i] == '\\')
|
||||
ret.replace(i, 1, "\\\\");
|
||||
else if(ret[i] == '\n')
|
||||
ret.replace(i, 1, "\\n");
|
||||
|
||||
i = ret.find_first_of("\"\n\\", i + 2);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::string AppVeyorListener::TestCase::MakeJSON()
|
||||
{
|
||||
std::string json;
|
||||
|
||||
return StringFormat::Fmt(
|
||||
R"({
|
||||
"testName": "%s",
|
||||
"testFramework": "Catch.hpp",
|
||||
"fileName": "%s",
|
||||
"outcome": "%s",
|
||||
"durationMilliseconds": "%.0f",
|
||||
"durationMilliseconds": "%d",
|
||||
"ErrorMessage": "%s",
|
||||
"ErrorStackTrace": "",
|
||||
"StdOut": "",
|
||||
"StdErr": ""
|
||||
})",
|
||||
testName.c_str(), fileName.c_str(), outcome, RDCMAX(msDuration, 0.0),
|
||||
escape(trim(errorList)).c_str());
|
||||
escape(name).c_str(), escape(filename).c_str(), passed ? "Passed" : "Failed",
|
||||
(int)RDCMAX(durationInSeconds * 1000.0, 0.0), escape(trim(errorList)).c_str());
|
||||
}
|
||||
|
||||
std::string http;
|
||||
http += StringFormat::Fmt("%s /api/tests HTTP/1.1\r\n", update ? "PUT" : "POST");
|
||||
http += StringFormat::Fmt("Host: %s\r\n", hostname.c_str());
|
||||
http += "Connection: close\r\n";
|
||||
http += "Content-Type: application/json\r\n";
|
||||
http += StringFormat::Fmt("Content-Length: %zu\r\n", json.size());
|
||||
http += "User-Agent: Catch.hpp appveyor updater\r\n";
|
||||
http += "\r\n";
|
||||
return http + json;
|
||||
}
|
||||
|
||||
std::string escape(const std::string &input)
|
||||
{
|
||||
std::string ret = input;
|
||||
size_t i = ret.find_first_of("\"\n\\", 0);
|
||||
while(i != std::string::npos)
|
||||
{
|
||||
if(ret[i] == '"')
|
||||
ret.replace(i, 1, "\\\"");
|
||||
else if(ret[i] == '\\')
|
||||
ret.replace(i, 1, "\\\\");
|
||||
else if(ret[i] == '\n')
|
||||
ret.replace(i, 1, "\\n");
|
||||
|
||||
i = ret.find_first_of("\"\n\\", i + 2);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
};
|
||||
CATCH_REGISTER_LISTENER(AppVeyorListener)
|
||||
|
||||
class LogOutputter : public std::stringbuf
|
||||
|
||||
Reference in New Issue
Block a user