From 21fcfa7758b83216fa5a823c3ed0eb6a66bdf0a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Villemot?= Date: Tue, 20 Dec 2022 14:47:32 +0100 Subject: [PATCH] use_dll: fixes to parallel compilation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit — No longer call std::exit() from threads when compilation fails, that function is marked as not thread-safe under GNU/Linux; and it leads to deadlocks under Windows. Rather store the list of failed objects, and exit with a message and an error code from the main thread when that list is not empty at the end of preprocessing. – Fix the condition used for waiting until all compilation threads finish; checking that the queue is empty is not enough, since a compilation may be ongoing. So also track objects whose compilation is ongoing. --- src/ModelTree.cc | 28 ++++++++++++++++++++-------- src/ModelTree.hh | 6 +++++- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/ModelTree.cc b/src/ModelTree.cc index 51060f45..1986485b 100644 --- a/src/ModelTree.cc +++ b/src/ModelTree.cc @@ -44,7 +44,7 @@ condition_variable_any ModelTree::mex_compilation_cv; mutex ModelTree::mex_compilation_mut; vector, string>> ModelTree::mex_compilation_queue; -set ModelTree::mex_compilation_done; +set ModelTree::mex_compilation_ongoing, ModelTree::mex_compilation_done, ModelTree::mex_compilation_failed; vector ModelTree::mex_compilation_workers; void @@ -1955,6 +1955,7 @@ ModelTree::initializeMEXCompilationWorkers(int numworkers) output = get<0>(*it); cmd = get<2>(*it); mex_compilation_queue.erase(it); + mex_compilation_ongoing.insert(output); return true; } return false; @@ -1964,13 +1965,13 @@ ModelTree::initializeMEXCompilationWorkers(int numworkers) if (mex_compilation_cv.wait(lk, stoken, pick_job)) { lk.unlock(); - if (system(cmd.c_str())) - { - cerr << "Compilation failed" << endl; - exit(EXIT_FAILURE); - } + int r { system(cmd.c_str()) }; lk.lock(); - mex_compilation_done.insert(output); + mex_compilation_ongoing.erase(output); + if (r) + mex_compilation_failed.insert(output); + else + mex_compilation_done.insert(output); /* The object just compiled may be a prerequisite for several other objects, so notify all waiting workers. Also needed to notify the main thread when in @@ -1984,7 +1985,18 @@ void ModelTree::waitForMEXCompilationWorkers() { unique_lock lk {mex_compilation_mut}; - mex_compilation_cv.wait(lk, [] { return mex_compilation_queue.empty(); }); + mex_compilation_cv.wait(lk, [] { + return (mex_compilation_queue.empty() && mex_compilation_ongoing.empty()) + || !mex_compilation_failed.empty(); }); + if (!mex_compilation_failed.empty()) + { + cerr << "Compilation failed for: "; + for (const auto &p : mex_compilation_failed) + cerr << p.string() << " "; + cerr << endl; + lk.unlock(); // So that threads can process their stoken + exit(EXIT_FAILURE); + } } void diff --git a/src/ModelTree.hh b/src/ModelTree.hh index 84a5d60b..a68bb22b 100644 --- a/src/ModelTree.hh +++ b/src/ModelTree.hh @@ -439,8 +439,12 @@ private: /* Object/MEX files waiting to be compiled (with their prerequisites as 2nd element and compilation command as the 3rd element) */ static vector, string>> mex_compilation_queue; - // Object/MEX files already compiled + // Object/MEX files in the process of being compiled + static set mex_compilation_ongoing; + // Object/MEX files already compiled successfully static set mex_compilation_done; + // Object/MEX files whose compilation failed + static set mex_compilation_failed; /* Compute a pseudo-Jacobian whose all elements are either zero or one, depending on whether the variable symbolically appears in the equation. If