From 78d974c4601787c1aa581883e81cf8d8d6b55683 Mon Sep 17 00:00:00 2001 From: Pavlo Hilei Date: Mon, 7 Oct 2024 16:37:15 +0200 Subject: [PATCH] Add OCCA_SOURCE_CACHE_DIR environment variable support. The purpose of this variable is to store source files cache in a separate folder than binary cache. This feature might be very handful for the projects that don't want to expose sources, but still utilize JIT. Source cache can be stored in some encrypted tmpfs for example. Also, this commit exposes occa::env::init so that application can change enviroment variables in a runtime, and make OCCA reread them. --- include/occa/utils/env.hpp | 4 ++- src/occa/internal/bin/occa.cpp | 7 +++-- src/occa/internal/core/launchedDevice.cpp | 3 +- src/occa/internal/io/cache.cpp | 15 ++++----- src/occa/internal/io/cache.hpp | 6 ++-- src/occa/internal/io/utils.cpp | 13 +++++--- src/occa/internal/io/utils.hpp | 1 + src/occa/internal/modes/serial/device.cpp | 3 +- src/occa/internal/utils/env.cpp | 37 +++++++++++++++++------ src/occa/internal/utils/env.hpp | 3 ++ tests/CMakeLists.txt | 3 ++ tests/src/internal/io/cache.cpp | 1 + 12 files changed, 67 insertions(+), 29 deletions(-) diff --git a/include/occa/utils/env.hpp b/include/occa/utils/env.hpp index 814a76b2f..aa846e96f 100644 --- a/include/occa/utils/env.hpp +++ b/include/occa/utils/env.hpp @@ -7,9 +7,11 @@ namespace occa { json& settings(); namespace env { - extern std::string OCCA_DIR, OCCA_INSTALL_DIR, OCCA_CACHE_DIR; + extern std::string OCCA_DIR, OCCA_INSTALL_DIR, OCCA_CACHE_DIR, OCCA_SOURCE_CACHE_DIR; void setOccaCacheDir(const std::string &path); + void setOccaBinaryCacheDir(const std::string &path); + void init(); } } diff --git a/src/occa/internal/bin/occa.cpp b/src/occa/internal/bin/occa.cpp index 15bd42437..800c90f4f 100644 --- a/src/occa/internal/bin/occa.cpp +++ b/src/occa/internal/bin/occa.cpp @@ -95,7 +95,8 @@ namespace occa { const bool promptCheck = !options["yes"]; if (options["all"] && - safeRmrf(env::OCCA_CACHE_DIR, promptCheck)) { + safeRmrf(env::OCCA_CACHE_DIR, promptCheck) && + safeRmrf(env::OCCA_SOURCE_CACHE_DIR, promptCheck)) { printRemovedMessage(true); return true; } @@ -103,9 +104,10 @@ namespace occa { bool removedSomething = false; if (options["kernels"]) { removedSomething |= safeRmrf(io::cachePath(), promptCheck); + removedSomething |= safeRmrf(io::sourceCachePath(), promptCheck); } if (options["locks"]) { - const std::string lockPath = env::OCCA_CACHE_DIR + "locks/"; + const std::string lockPath = env::OCCA_SOURCE_CACHE_DIR + "locks/"; removedSomething |= safeRmrf(lockPath, promptCheck); } @@ -240,6 +242,7 @@ namespace occa { io::stdout << " Basic:\n" << " - OCCA_DIR : " << envEcho("OCCA_DIR") << "\n" << " - OCCA_CACHE_DIR : " << envEcho("OCCA_CACHE_DIR") << "\n" + << " - OCCA_SOURCE_CACHE_DIR : " << envEcho("OCCA_SOURCE_CACHE_DIR") << "\n" << " - OCCA_VERBOSE : " << envEcho("OCCA_VERBOSE") << "\n" << " - OCCA_UNSAFE : " << OCCA_UNSAFE << "\n" diff --git a/src/occa/internal/core/launchedDevice.cpp b/src/occa/internal/core/launchedDevice.cpp index 821f97626..841c54dcf 100644 --- a/src/occa/internal/core/launchedDevice.cpp +++ b/src/occa/internal/core/launchedDevice.cpp @@ -91,8 +91,9 @@ namespace occa { const bool usingOkl, const occa::json &kernelProps) { const std::string hashDir = io::hashDir(filename, kernelHash); + const std::string binaryHashDir = io::hashDir(filename, kernelHash, true); std::string sourceFilename = hashDir + kc::cachedSourceFilename(filename); - const std::string binaryFilename = hashDir + kc::binaryFile; + const std::string binaryFilename = binaryHashDir + kc::binaryFile; // Check if binary exists and is finished const bool foundBinary = io::isFile(binaryFilename); diff --git a/src/occa/internal/io/cache.cpp b/src/occa/internal/io/cache.cpp index 59c360c64..ff0079a03 100644 --- a/src/occa/internal/io/cache.cpp +++ b/src/occa/internal/io/cache.cpp @@ -9,7 +9,7 @@ namespace occa { namespace io { - bool isCached(const std::string &filename) { + bool isCached(const std::string &filename, bool binary) { // Directory, not file if (filename.size() == 0) { return false; @@ -18,19 +18,20 @@ namespace occa { std::string expFilename = io::expandFilename(filename); // File is already cached - const std::string &cPath = cachePath(); + const std::string &cPath = binary ? cachePath() : sourceCachePath(); return startsWith(expFilename, cPath); } - std::string hashDir(const hash_t &hash) { - return hashDir("", hash); + std::string hashDir(const hash_t &hash, bool binary) { + return hashDir("", hash, binary); } std::string hashDir(const std::string &filename, - const hash_t &hash) { - bool fileIsCached = isCached(filename); + const hash_t &hash, + bool binary) { + bool fileIsCached = isCached(filename, binary); - const std::string &cPath = cachePath(); + const std::string &cPath = binary ? cachePath() : sourceCachePath(); std::string cacheDir = cPath; const bool useHash = !filename.size() || !fileIsCached; diff --git a/src/occa/internal/io/cache.hpp b/src/occa/internal/io/cache.hpp index c3a0a130f..4d411cbf0 100644 --- a/src/occa/internal/io/cache.hpp +++ b/src/occa/internal/io/cache.hpp @@ -9,12 +9,12 @@ namespace occa { class json; namespace io { - bool isCached(const std::string &filename); + bool isCached(const std::string &filename, bool binary=false); - std::string hashDir(const hash_t &hash); + std::string hashDir(const hash_t &hash, bool binary=false); std::string hashDir(const std::string &filename, - const hash_t &hash = hash_t()); + const hash_t &hash = hash_t(), bool binary=false); std::string cacheFile(const std::string &filename, const std::string &cachedName, diff --git a/src/occa/internal/io/utils.cpp b/src/occa/internal/io/utils.cpp index b1ee58748..ca3a84d2e 100644 --- a/src/occa/internal/io/utils.cpp +++ b/src/occa/internal/io/utils.cpp @@ -65,6 +65,10 @@ namespace occa { return env::OCCA_CACHE_DIR + "cache/"; } + std::string sourceCachePath() { + return env::OCCA_SOURCE_CACHE_DIR + "cache/"; + } + std::string libraryPath() { return env::OCCA_CACHE_DIR + "libraries/"; } @@ -290,12 +294,11 @@ namespace occa { std::string shortname(const std::string &filename) { std::string expFilename = io::expandFilename(filename); - if (!startsWith(expFilename, env::OCCA_CACHE_DIR)) { - return filename; + if (startsWith(expFilename, env::OCCA_CACHE_DIR) || startsWith(expFilename, env::OCCA_SOURCE_CACHE_DIR)) { + const std::string &cPath = startsWith(expFilename, env::OCCA_CACHE_DIR) ? cachePath() : sourceCachePath(); + return expFilename.substr(cPath.size()); } - - const std::string &cPath = cachePath(); - return expFilename.substr(cPath.size()); + return filename; } std::string findInPaths(const std::string &filename, const strVector &paths) { diff --git a/src/occa/internal/io/utils.hpp b/src/occa/internal/io/utils.hpp index 9747a3729..ecb2d938c 100644 --- a/src/occa/internal/io/utils.hpp +++ b/src/occa/internal/io/utils.hpp @@ -24,6 +24,7 @@ namespace occa { typedef std::map libraryPathMap_t; std::string cachePath(); + std::string sourceCachePath(); std::string libraryPath(); std::string currentWorkingDirectory(); diff --git a/src/occa/internal/modes/serial/device.cpp b/src/occa/internal/modes/serial/device.cpp index 97376e0ec..a7ea98644 100644 --- a/src/occa/internal/modes/serial/device.cpp +++ b/src/occa/internal/modes/serial/device.cpp @@ -121,13 +121,14 @@ namespace occa { const occa::json &kernelProps, const bool isLauncherKernel) { const std::string hashDir = io::hashDir(filename, kernelHash); + const std::string binaryHashDir = io::hashDir(filename, kernelHash, true); const std::string &kcBinaryFile = ( isLauncherKernel ? kc::launcherBinaryFile : kc::binaryFile ); - std::string binaryFilename = hashDir + kcBinaryFile; + std::string binaryFilename = binaryHashDir + kcBinaryFile; // Check if binary exists and is finished const bool foundBinary = io::isFile(binaryFilename); diff --git a/src/occa/internal/utils/env.cpp b/src/occa/internal/utils/env.cpp index 5ee810a3e..ad6c9fe39 100644 --- a/src/occa/internal/utils/env.cpp +++ b/src/occa/internal/utils/env.cpp @@ -32,7 +32,7 @@ namespace occa { std::string HOME, CWD; std::string PATH, LD_LIBRARY_PATH; - std::string OCCA_DIR, OCCA_INSTALL_DIR, OCCA_CACHE_DIR; + std::string OCCA_DIR, OCCA_INSTALL_DIR, OCCA_CACHE_DIR, OCCA_SOURCE_CACHE_DIR; size_t OCCA_MEM_BYTE_ALIGN; strVector OCCA_INCLUDE_PATH; strVector OCCA_LIBRARY_PATH; @@ -58,12 +58,13 @@ namespace occa { envInitializer_t::setupCachePath(); } - envInitializer_t::envInitializer_t() : - isInitialized(false) { - if (isInitialized) { - return; - } + void setOccaSourceCacheDir(const std::string &path) { + OCCA_SOURCE_CACHE_DIR = path; + envInitializer_t::setupCachePath(); + } + + void envInitializer_t::init() { initSettings(); initEnvironment(); loadConfig(); @@ -73,6 +74,14 @@ namespace occa { isInitialized = true; } + envInitializer_t::envInitializer_t() : + isInitialized(false) { + if (isInitialized) { + return; + } + init(); + } + void envInitializer_t::initSettings() { json &settings_ = baseSettings(); settings_["version"] = OCCA_VERSION_STR; @@ -94,8 +103,9 @@ namespace occa { PATH = env::var("PATH"); LD_LIBRARY_PATH = env::var("LD_LIBRARY_PATH"); - OCCA_CACHE_DIR = env::var("OCCA_CACHE_DIR"); - OCCA_COLOR_ENABLED = env::get("OCCA_COLOR_ENABLED", true); + OCCA_CACHE_DIR = env::var("OCCA_CACHE_DIR"); + OCCA_SOURCE_CACHE_DIR = env::var("OCCA_SOURCE_CACHE_DIR"); + OCCA_COLOR_ENABLED = env::get("OCCA_COLOR_ENABLED", true); OCCA_INCLUDE_PATH = split(env::var("OCCA_INCLUDE_PATH"), ':', '\\'); OCCA_LIBRARY_PATH = split(env::var("OCCA_LIBRARY_PATH"), ':', '\\'); @@ -124,6 +134,7 @@ namespace occa { io::endWithSlash(OCCA_DIR); io::endWithSlash(OCCA_INSTALL_DIR); io::endWithSlash(OCCA_CACHE_DIR); + io::endWithSlash(OCCA_SOURCE_CACHE_DIR); OCCA_MEM_BYTE_ALIGN = OCCA_DEFAULT_MEM_BYTE_ALIGN; if (env::var("OCCA_MEM_BYTE_ALIGN").size() > 0) { @@ -142,7 +153,7 @@ namespace occa { void envInitializer_t::loadConfig() { const std::string configFile = ( env::get("OCCA_CONFIG", - OCCA_CACHE_DIR + "config.json") + OCCA_SOURCE_CACHE_DIR + "config.json") ); if (!io::exists(configFile)) { @@ -179,10 +190,18 @@ namespace occa { if (!io::isDir(env::OCCA_CACHE_DIR)) { sys::mkpath(env::OCCA_CACHE_DIR); } + + if (env::OCCA_SOURCE_CACHE_DIR.size() == 0) { + env::OCCA_SOURCE_CACHE_DIR = env::OCCA_CACHE_DIR; + } } envInitializer_t::~envInitializer_t() {} envInitializer_t envInitializer; + + void init() { + envInitializer.init(); + } } } diff --git a/src/occa/internal/utils/env.hpp b/src/occa/internal/utils/env.hpp index 241b384b2..597374207 100644 --- a/src/occa/internal/utils/env.hpp +++ b/src/occa/internal/utils/env.hpp @@ -33,11 +33,13 @@ namespace occa { } void setOccaCacheDir(const std::string &path); + void setOccaSourceCacheDir(const std::string &path); class envInitializer_t { public: envInitializer_t(); ~envInitializer_t(); + void init(); private: bool isInitialized; @@ -53,6 +55,7 @@ namespace occa { static void cleanFileOpeners(); friend void setOccaCacheDir(const std::string &path); + friend void setOccaSourceCacheDir(const std::string &path); }; extern envInitializer_t envInitializer; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6f8b8ffdc..ab2ff6d51 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -32,6 +32,9 @@ function(add_occa_test test_source) # Expected defines set_property(TEST ${cmake_test_target} APPEND PROPERTY ENVIRONMENT OCCA_CACHE_DIR=${OCCA_BUILD_DIR}/occa) + + set_property(TEST ${cmake_test_target} APPEND PROPERTY + ENVIRONMENT OCCA_SOURCE_CACHE_DIR=${OCCA_BUILD_DIR}/occa) endfunction() #---[ Setup Tests ]--------------------- diff --git a/tests/src/internal/io/cache.cpp b/tests/src/internal/io/cache.cpp index 31af9fdaf..dba4801ac 100644 --- a/tests/src/internal/io/cache.cpp +++ b/tests/src/internal/io/cache.cpp @@ -12,6 +12,7 @@ void testBuild(); int main(const int argc, const char **argv) { #ifndef USE_CMAKE occa::env::OCCA_CACHE_DIR = occa::io::dirname(__FILE__); + occa::env::OCCA_SOURCE_CACHE_DIR = occa::env::OCCA_CACHE_DIR; #endif srand(time(NULL));