diff --git a/CMakeLists.txt b/CMakeLists.txt index 977239b8b..501ccccc2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -269,17 +269,26 @@ endif() add_subdirectory(renderdoc) -if(ENABLE_RENDERDOCCMD) - add_subdirectory(renderdoccmd) -endif() - # these variables are handled within the CMakeLists.txt in qrenderdoc, # but we need to add it if either is enabled since the swig bindings # are handled in common if(ENABLE_QRENDERDOC OR ENABLE_PYRENDERDOC) + # Make sure Python 3 is found + set(Python_ADDITIONAL_VERSIONS 3.4 3.5 3.6 3.7) + find_package(PythonInterp 3 REQUIRED) + find_package(PythonLibs 3 REQUIRED) + # we also need python3-config for swig + if(NOT EXISTS "${PYTHON_EXECUTABLE}-config" AND NOT EXISTS "${PYTHON_EXECUTABLE}.${PYTHON_VERSION_MINOR}-config") + message(FATAL_ERROR "We require ${PYTHON_EXECUTABLE}-config or ${PYTHON_EXECUTABLE}.${PYTHON_VERSION_MINOR}-config to build swig, please install the python dev package for your system.") + endif() + add_subdirectory(qrenderdoc) endif() +if(ENABLE_RENDERDOCCMD) + add_subdirectory(renderdoccmd) +endif() + # install documentation files install (FILES util/LINUX_DIST_README DESTINATION share/doc/renderdoc RENAME README) install (FILES LICENSE.md DESTINATION share/doc/renderdoc) diff --git a/qrenderdoc/CMakeLists.txt b/qrenderdoc/CMakeLists.txt index 5a93ce8a8..3da51eeaf 100644 --- a/qrenderdoc/CMakeLists.txt +++ b/qrenderdoc/CMakeLists.txt @@ -60,15 +60,6 @@ else() add_custom_command(OUTPUT RenderDoc.icns COMMAND touch RenderDoc.icns) endif() -# Make sure Python 3 is found -set(Python_ADDITIONAL_VERSIONS 3.4 3.5 3.6 3.7) -find_package(PythonInterp 3 REQUIRED) -find_package(PythonLibs 3 REQUIRED) -# we also need python3-config for swig -if(NOT EXISTS "${PYTHON_EXECUTABLE}-config" AND NOT EXISTS "${PYTHON_EXECUTABLE}.${PYTHON_VERSION_MINOR}-config") - message(FATAL_ERROR "We require ${PYTHON_EXECUTABLE}-config or ${PYTHON_EXECUTABLE}.${PYTHON_VERSION_MINOR}-config to build swig, please install the python dev package for your system.") -endif() - include(ExternalProject) # Need bison for swig diff --git a/renderdoc/api/replay/renderdoc_replay.h b/renderdoc/api/replay/renderdoc_replay.h index 28f672e70..ae73cdfcd 100644 --- a/renderdoc/api/replay/renderdoc_replay.h +++ b/renderdoc/api/replay/renderdoc_replay.h @@ -2291,3 +2291,7 @@ extern "C" RENDERDOC_API AndroidFlags RENDERDOC_CC RENDERDOC_MakeDebuggablePacka DOCUMENT("Internal function that runs unit tests."); extern "C" RENDERDOC_API int RENDERDOC_CC RENDERDOC_RunUnitTests(const rdcstr &command, const rdcarray &args); + +DOCUMENT("Internal function that runs functional tests."); +extern "C" RENDERDOC_API int RENDERDOC_CC RENDERDOC_RunFunctionalTests(int pythonMinorVersion, + const rdcarray &args); diff --git a/renderdoc/os/os_specific.h b/renderdoc/os/os_specific.h index 8f91b41b8..cffc867fe 100644 --- a/renderdoc/os/os_specific.h +++ b/renderdoc/os/os_specific.h @@ -370,7 +370,8 @@ namespace StringFormat { void sntimef(time_t utcTime, char *str, size_t bufSize, const char *format); -string Wide2UTF8(const std::wstring &s); +std::string Wide2UTF8(const std::wstring &s); +std::wstring UTF82Wide(const std::string &s); void Shutdown(); }; diff --git a/renderdoc/os/posix/android/android_stringio.cpp b/renderdoc/os/posix/android/android_stringio.cpp index aee04b4c0..753dcfe4d 100644 --- a/renderdoc/os/posix/android/android_stringio.cpp +++ b/renderdoc/os/posix/android/android_stringio.cpp @@ -102,12 +102,18 @@ void GetLibraryFilename(string &selfName) namespace StringFormat { -string Wide2UTF8(const std::wstring &s) +std::string Wide2UTF8(const std::wstring &s) { RDCFATAL("Converting wide strings to UTF-8 is not supported on Android!"); return ""; } +std::wstring UTF82Wide(const std::string &s) +{ + RDCFATAL("Converting UTF-8 to wide strings is not supported on Android!"); + return L""; +} + void Shutdown() { } diff --git a/renderdoc/os/posix/apple/apple_stringio.cpp b/renderdoc/os/posix/apple/apple_stringio.cpp index d37c0f503..52346bdb1 100644 --- a/renderdoc/os/posix/apple/apple_stringio.cpp +++ b/renderdoc/os/posix/apple/apple_stringio.cpp @@ -134,35 +134,38 @@ namespace StringFormat { // cache iconv_t descriptor to save on iconv_open/iconv_close each time iconv_t iconvWide2UTF8 = (iconv_t)-1; +iconv_t iconvUTF82Wide = (iconv_t)-1; // iconv is not thread safe when sharing an iconv_t descriptor // I don't expect much contention but if it happens we could TryLock // before creating a temporary iconv_t, or hold two iconv_ts, or something. -Threading::CriticalSection lockWide2UTF8; +Threading::CriticalSection iconvLock; void Shutdown() { - SCOPED_LOCK(lockWide2UTF8); + SCOPED_LOCK(iconvLock); + if(iconvWide2UTF8 != (iconv_t)-1) iconv_close(iconvWide2UTF8); iconvWide2UTF8 = (iconv_t)-1; + + if(iconvUTF82Wide != (iconv_t)-1) + iconv_close(iconvUTF82Wide); + iconvUTF82Wide = (iconv_t)-1; } -string Wide2UTF8(const std::wstring &s) +std::string Wide2UTF8(const std::wstring &s) { // include room for null terminator, assuming unicode input (not ucs) // utf-8 characters can be max 4 bytes. size_t len = (s.length() + 1) * 4; - vector charBuffer; - - if(charBuffer.size() < len) - charBuffer.resize(len); + std::vector charBuffer(len); size_t ret; { - SCOPED_LOCK(lockWide2UTF8); + SCOPED_LOCK(iconvLock); if(iconvWide2UTF8 == (iconv_t)-1) iconvWide2UTF8 = iconv_open("UTF-8", "WCHAR_T"); @@ -192,7 +195,49 @@ string Wide2UTF8(const std::wstring &s) // convert to string from null-terminated string - utf-8 never contains // 0 bytes before the null terminator, and this way we don't care if // charBuffer is larger than the string - return string(&charBuffer[0]); + return std::string(&charBuffer[0]); +} + +std::wstring UTF82Wide(const std::string &s) +{ + // include room for null terminator, for ascii input we need at least as many output chars as + // input. + size_t len = s.length() + 1; + + std::vector wcharBuffer(len); + + size_t ret; + + { + SCOPED_LOCK(iconvLock); + + if(iconvUTF82Wide == (iconv_t)-1) + iconvUTF82Wide = iconv_open("WCHAR_T", "UTF-8"); + + if(iconvUTF82Wide == (iconv_t)-1) + { + RDCERR("Couldn't open iconv for UTF-8 to WCHAR_T: %d", errno); + return L""; + } + + char *inbuf = (char *)s.c_str(); + size_t insize = s.length() + 1; // include null terminator + char *outbuf = (char *)&wcharBuffer[0]; + size_t outsize = len * sizeof(wchar_t); + + ret = iconv(iconvUTF82Wide, &inbuf, &insize, &outbuf, &outsize); + } + + if(ret == (size_t)-1) + { +#if ENABLED(RDOC_DEVEL) + RDCWARN("Failed to convert wstring"); +#endif + return L""; + } + + // convert to string from null-terminated string + return std::wstring(&wcharBuffer[0]); } }; diff --git a/renderdoc/os/posix/linux/linux_stringio.cpp b/renderdoc/os/posix/linux/linux_stringio.cpp index 4307fa660..c63333622 100644 --- a/renderdoc/os/posix/linux/linux_stringio.cpp +++ b/renderdoc/os/posix/linux/linux_stringio.cpp @@ -389,34 +389,38 @@ namespace StringFormat { // cache iconv_t descriptor to save on iconv_open/iconv_close each time iconv_t iconvWide2UTF8 = (iconv_t)-1; +iconv_t iconvUTF82Wide = (iconv_t)-1; // iconv is not thread safe when sharing an iconv_t descriptor // I don't expect much contention but if it happens we could TryLock // before creating a temporary iconv_t, or hold two iconv_ts, or something. -Threading::CriticalSection lockWide2UTF8; +Threading::CriticalSection iconvLock; void Shutdown() { - SCOPED_LOCK(lockWide2UTF8); - iconv_close(iconvWide2UTF8); + SCOPED_LOCK(iconvLock); + + if(iconvWide2UTF8 != (iconv_t)-1) + iconv_close(iconvWide2UTF8); iconvWide2UTF8 = (iconv_t)-1; + + if(iconvUTF82Wide != (iconv_t)-1) + iconv_close(iconvUTF82Wide); + iconvUTF82Wide = (iconv_t)-1; } -string Wide2UTF8(const std::wstring &s) +std::string Wide2UTF8(const std::wstring &s) { // include room for null terminator, assuming unicode input (not ucs) // utf-8 characters can be max 4 bytes. size_t len = (s.length() + 1) * 4; - vector charBuffer; - - if(charBuffer.size() < len) - charBuffer.resize(len); + std::vector charBuffer(len); size_t ret; { - SCOPED_LOCK(lockWide2UTF8); + SCOPED_LOCK(iconvLock); if(iconvWide2UTF8 == (iconv_t)-1) iconvWide2UTF8 = iconv_open("UTF-8", "WCHAR_T"); @@ -446,7 +450,49 @@ string Wide2UTF8(const std::wstring &s) // convert to string from null-terminated string - utf-8 never contains // 0 bytes before the null terminator, and this way we don't care if // charBuffer is larger than the string - return string(&charBuffer[0]); + return std::string(&charBuffer[0]); +} + +std::wstring UTF82Wide(const std::string &s) +{ + // include room for null terminator, for ascii input we need at least as many output chars as + // input. + size_t len = s.length() + 1; + + std::vector wcharBuffer(len); + + size_t ret; + + { + SCOPED_LOCK(iconvLock); + + if(iconvUTF82Wide == (iconv_t)-1) + iconvUTF82Wide = iconv_open("WCHAR_T", "UTF-8"); + + if(iconvUTF82Wide == (iconv_t)-1) + { + RDCERR("Couldn't open iconv for UTF-8 to WCHAR_T: %d", errno); + return L""; + } + + char *inbuf = (char *)s.c_str(); + size_t insize = s.length() + 1; // include null terminator + char *outbuf = (char *)&wcharBuffer[0]; + size_t outsize = len * sizeof(wchar_t); + + ret = iconv(iconvUTF82Wide, &inbuf, &insize, &outbuf, &outsize); + } + + if(ret == (size_t)-1) + { +#if ENABLED(RDOC_DEVEL) + RDCWARN("Failed to convert wstring"); +#endif + return L""; + } + + // convert to string from null-terminated string + return std::wstring(&wcharBuffer[0]); } }; diff --git a/renderdoc/os/win32/win32_specific.h b/renderdoc/os/win32/win32_specific.h index 43b6664a2..74e72e91b 100644 --- a/renderdoc/os/win32/win32_specific.h +++ b/renderdoc/os/win32/win32_specific.h @@ -47,12 +47,6 @@ #define GetEmbeddedResource(filename) GetDynamicEmbeddedResource(EmbeddedResource(filename)) std::string GetDynamicEmbeddedResource(int resource); -namespace StringFormat -{ -// useful for converting to wide before passing to OS functions -std::wstring UTF82Wide(const string &s); -}; - namespace OSUtility { inline void ForceCrash() diff --git a/renderdoc/replay/entry_points.cpp b/renderdoc/replay/entry_points.cpp index 6d5015f9d..9bfa42017 100644 --- a/renderdoc/replay/entry_points.cpp +++ b/renderdoc/replay/entry_points.cpp @@ -804,3 +804,113 @@ extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_ResourceFormatName(const Re { name = ResourceFormatName(fmt); } + +static void TestPrintMsg(const std::string &msg) +{ + OSUtility::WriteOutput(OSUtility::Output_DebugMon, msg.c_str()); + OSUtility::WriteOutput(OSUtility::Output_StdErr, msg.c_str()); +} + +extern "C" RENDERDOC_API int RENDERDOC_CC RENDERDOC_RunFunctionalTests(int pythonMinorVersion, + const rdcarray &args) +{ +#if ENABLED(RDOC_WIN32) + const char *moduledir = "/pymodules"; + const char *modulename = "renderdoc.pyd"; + std::string pythonlibs[] = {"python3?.dll"}; +#elif ENABLED(RDOC_LINUX) + const char *moduledir = ""; + const char *modulename = "renderdoc.so"; + // we don't care about pymalloc or not + std::string pythonlibs[] = {"libpython3.?m.so.1.0", "libpython3.?.so.1.0", "libpython3.?m.so", + "libpython3.?.so"}; +#else + const char *moduledir = ""; + const char *modulename = ""; + std::string pythonlibs[] = {}; + TestPrintMsg( + "Running functional tests not directly supported on this platform.\n" + "Try running util/test/run_tests.py manually.\n"); + return 1; +#endif + + std::string libPath; + FileIO::GetLibraryFilename(libPath); + + libPath = dirname(libPath); + std::string modulePath = libPath + moduledir; + + std::string moduleFilename = modulePath + "/" + modulename; + + if(!FileIO::exists(moduleFilename.c_str())) + { + TestPrintMsg(StringFormat::Fmt("Couldn't locate python module at %s\n", moduleFilename.c_str())); + return 1; + } + + // if we've been built either on windows or on linux from within the project root, going up two + // directories from the library will put us at the project root. This is the most common scenario + // and we don't add handling for locating the script elsewhere as in that case the user can run it + // directly. This is just intended as a useful shortcut for common cases. + std::string scriptPath = libPath + "/../../util/test/run_tests.py"; + + if(!FileIO::exists(scriptPath.c_str())) + { + TestPrintMsg(StringFormat::Fmt("Couldn't locate run_tests.py script at %s\n", scriptPath.c_str())); + return 1; + } + + void *handle = NULL; + + for(std::string py : pythonlibs) + { + // patch up the python minor version + char *ver = strchr(&py[0], '?'); + *ver = char('0' + pythonMinorVersion); + + handle = Process::LoadModule(py.c_str()); + RDCLOG("Loaded python from %s", py.c_str()); + } + + if(!handle) + { + TestPrintMsg("Couldn't locate python 3.6 library\n"); + return 1; + } + + typedef int(RENDERDOC_CC * PFN_Py_Main)(int, wchar_t **); + + PFN_Py_Main mainFunc = (PFN_Py_Main)Process::GetFunctionAddress(handle, "Py_Main"); + + if(!mainFunc) + { + TestPrintMsg("Couldn't get Py_Main in python library\n"); + return 1; + } + + std::vector wideArgs(args.size()); + + for(size_t i = 0; i < args.size(); i++) + wideArgs[i] = StringFormat::UTF82Wide(args[i]); + + // insert fake arguments to point at the script and our modules + wideArgs.insert(wideArgs.begin(), + { + L"python", + // specify script path + StringFormat::UTF82Wide(scriptPath), + // specify native library path + L"--renderdoc", StringFormat::UTF82Wide(libPath), + // specify python module path + L"--pyrenderdoc", StringFormat::UTF82Wide(modulePath), + // force in-process as we can't fork out to python to pass args + L"--in-process", + }); + + std::vector wideArgStrings(wideArgs.size()); + + for(size_t i = 0; i < wideArgs.size(); i++) + wideArgStrings[i] = &wideArgs[i][0]; + + return mainFunc((int)wideArgStrings.size(), wideArgStrings.data()); +} diff --git a/renderdoccmd/CMakeLists.txt b/renderdoccmd/CMakeLists.txt index b783e233f..3b2ff3715 100644 --- a/renderdoccmd/CMakeLists.txt +++ b/renderdoccmd/CMakeLists.txt @@ -2,6 +2,12 @@ set(sources renderdoccmd.cpp ${CMAKE_SOURCE_DIR}/renderdoc/api/replay/version.cp set(includes PRIVATE ${CMAKE_SOURCE_DIR}/renderdoc/api) set(libraries PRIVATE renderdoc) +if(PythonLibs_FOUND) + add_definitions(-DPYTHON_VERSION_MINOR=${PYTHON_VERSION_MINOR}) +else() + add_definitions(-DPYTHON_VERSION_MINOR=0) +endif() + if(APPLE) list(APPEND sources renderdoccmd_apple.cpp) elseif(ANDROID) diff --git a/renderdoccmd/renderdoccmd.cpp b/renderdoccmd/renderdoccmd.cpp index 2d058c3a0..fb016eb2a 100644 --- a/renderdoccmd/renderdoccmd.cpp +++ b/renderdoccmd/renderdoccmd.cpp @@ -799,7 +799,13 @@ struct TestCommand : public Command TestCommand(const GlobalEnvironment &env) : Command(env) {} virtual void AddOptions(cmdline::parser &parser) { - parser.set_footer(" [... parameters to test framework ...]"); + parser.set_footer( +#if PYTHON_MINOR_VERSION > 0 + "" +#else + "" +#endif + " [... parameters to test framework ...]"); parser.add("help", '\0', "print this message"); parser.stop_at_rest(true); } @@ -826,6 +832,10 @@ struct TestCommand : public Command if(mode == "unit") return RENDERDOC_RunUnitTests("renderdoccmd test unit", convertArgs(rest)); +#if PYTHON_MINOR_VERSION > 0 + else if(mode == "functional") + return RENDERDOC_RunFunctionalTests(PYTHON_MINOR_VERSION, convertArgs(rest)); +#endif std::cerr << "Unsupported test frame work '" << mode << "'" << std::endl << std::endl; std::cerr << parser.usage() << std::endl; diff --git a/renderdoccmd/renderdoccmd.vcxproj b/renderdoccmd/renderdoccmd.vcxproj index 55238c332..82cfa13c9 100644 --- a/renderdoccmd/renderdoccmd.vcxproj +++ b/renderdoccmd/renderdoccmd.vcxproj @@ -100,7 +100,7 @@ Level3 Disabled - WIN32;_CRT_SECURE_NO_WARNINGS;RENDERDOC_PLATFORM_WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + WIN32;_CRT_SECURE_NO_WARNINGS;RENDERDOC_PLATFORM_WIN32;NDEBUG;_CONSOLE;PYTHON_VERSION_MINOR=6;%(PreprocessorDefinitions) $(SolutionDir)renderdocshim\;$(SolutionDir)renderdoc\api\;$(SolutionDir)renderdoc\api\replay;$(SolutionDir)renderdoc\3rdparty\ MultiThreadedDLL true @@ -121,7 +121,7 @@ Level3 Disabled - WIN32;_CRT_SECURE_NO_WARNINGS;WIN64;RENDERDOC_PLATFORM_WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + WIN32;_CRT_SECURE_NO_WARNINGS;WIN64;RENDERDOC_PLATFORM_WIN32;NDEBUG;_CONSOLE;PYTHON_VERSION_MINOR=6;%(PreprocessorDefinitions) $(SolutionDir)renderdocshim\;$(SolutionDir)renderdoc\api\;$(SolutionDir)renderdoc\api\replay;$(SolutionDir)renderdoc\3rdparty\ MultiThreadedDLL true @@ -144,7 +144,7 @@ MaxSpeed true true - WIN32;_CRT_SECURE_NO_WARNINGS;RENDERDOC_PLATFORM_WIN32;NDEBUG;RELEASE;_CONSOLE;%(PreprocessorDefinitions) + WIN32;_CRT_SECURE_NO_WARNINGS;RENDERDOC_PLATFORM_WIN32;NDEBUG;RELEASE;_CONSOLE;PYTHON_VERSION_MINOR=6;%(PreprocessorDefinitions) $(SolutionDir)renderdocshim\;$(SolutionDir)renderdoc\api\;$(SolutionDir)renderdoc\api\replay;$(SolutionDir)renderdoc\3rdparty\ true @@ -167,7 +167,7 @@ MaxSpeed true true - WIN32;_CRT_SECURE_NO_WARNINGS;WIN64;RENDERDOC_PLATFORM_WIN32;NDEBUG;RELEASE;_CONSOLE;%(PreprocessorDefinitions) + WIN32;_CRT_SECURE_NO_WARNINGS;WIN64;RENDERDOC_PLATFORM_WIN32;NDEBUG;RELEASE;_CONSOLE;PYTHON_VERSION_MINOR=6;%(PreprocessorDefinitions) $(SolutionDir)renderdocshim\;$(SolutionDir)renderdoc\api\;$(SolutionDir)renderdoc\api\replay;$(SolutionDir)renderdoc\3rdparty\ true