JezK
Edit File: greenlet_thread_state.hpp
#ifndef GREENLET_THREAD_STATE_HPP #define GREENLET_THREAD_STATE_HPP #include <ctime> #include <stdexcept> #include "greenlet_internal.hpp" #include "greenlet_refs.hpp" #include "greenlet_thread_support.hpp" using greenlet::refs::BorrowedObject; using greenlet::refs::BorrowedGreenlet; using greenlet::refs::BorrowedMainGreenlet; using greenlet::refs::OwnedMainGreenlet; using greenlet::refs::OwnedObject; using greenlet::refs::OwnedGreenlet; using greenlet::refs::OwnedList; using greenlet::refs::PyErrFetchParam; using greenlet::refs::PyArgParseParam; using greenlet::refs::ImmortalString; using greenlet::refs::CreatedModule; using greenlet::refs::PyErrPieces; using greenlet::refs::NewReference; namespace greenlet { /** * Thread-local state of greenlets. * * Each native thread will get exactly one of these objects, * automatically accessed through the best available thread-local * mechanism the compiler supports (``thread_local`` for C++11 * compilers or ``__thread``/``declspec(thread)`` for older GCC/clang * or MSVC, respectively.) * * Previously, we kept thread-local state mostly in a bunch of * ``static volatile`` variables in the main greenlet file.. This had * the problem of requiring extra checks, loops, and great care * accessing these variables if we potentially invoked any Python code * that could release the GIL, because the state could change out from * under us. Making the variables thread-local solves this problem. * * When we detected that a greenlet API accessing the current greenlet * was invoked from a different thread than the greenlet belonged to, * we stored a reference to the greenlet in the Python thread * dictionary for the thread the greenlet belonged to. This could lead * to memory leaks if the thread then exited (because of a reference * cycle, as greenlets referred to the thread dictionary, and deleting * non-current greenlets leaked their frame plus perhaps arguments on * the C stack). If a thread exited while still having running * greenlet objects (perhaps that had just switched back to the main * greenlet), and did not invoke one of the greenlet APIs *in that * thread, immediately before it exited, without some other thread * then being invoked*, such a leak was guaranteed. * * This can be partly solved by using compiler thread-local variables * instead of the Python thread dictionary, thus avoiding a cycle. * * To fully solve this problem, we need a reliable way to know that a * thread is done and we should clean up the main greenlet. On POSIX, * we can use the destructor function of ``pthread_key_create``, but * there's nothing similar on Windows; a C++11 thread local object * reliably invokes its destructor when the thread it belongs to exits * (non-C++11 compilers offer ``__thread`` or ``declspec(thread)`` to * create thread-local variables, but they can't hold C++ objects that * invoke destructors; the C++11 version is the most portable solution * I found). When the thread exits, we can drop references and * otherwise manipulate greenlets and frames that we know can no * longer be switched to. For compilers that don't support C++11 * thread locals, we have a solution that uses the python thread * dictionary, though it may not collect everything as promptly as * other compilers do, if some other library is using the thread * dictionary and has a cycle or extra reference. * * There are two small wrinkles. The first is that when the thread * exits, it is too late to actually invoke Python APIs: the Python * thread state is gone, and the GIL is released. To solve *this* * problem, our destructor uses ``Py_AddPendingCall`` to transfer the * destruction work to the main thread. (This is not an issue for the * dictionary solution.) * * The second is that once the thread exits, the thread local object * is invalid and we can't even access a pointer to it, so we can't * pass it to ``Py_AddPendingCall``. This is handled by actually using * a second object that's thread local (ThreadStateCreator) and having * it dynamically allocate this object so it can live until the * pending call runs. */ class ThreadState { private: // As of commit 08ad1dd7012b101db953f492e0021fb08634afad // this class needed 56 bytes in o Py_DEBUG build // on 64-bit macOS 11. // Adding the vector takes us up to 80 bytes () /* Strong reference to the main greenlet */ OwnedMainGreenlet main_greenlet; /* Strong reference to the current greenlet. */ OwnedGreenlet current_greenlet; /* Strong reference to the trace function, if any. */ OwnedObject tracefunc; typedef std::vector<PyGreenlet*, PythonAllocator<PyGreenlet*> > deleteme_t; /* A vector of raw PyGreenlet pointers representing things that need deleted when this thread is running. The vector owns the references, but you need to manually INCREF/DECREF as you use them. We don't use a vector<refs::OwnedGreenlet> because we make copy of this vector, and that would become O(n) as all the refcounts are incremented in the copy. */ deleteme_t deleteme; #ifdef GREENLET_NEEDS_EXCEPTION_STATE_SAVED void* exception_state; #endif static std::clock_t _clocks_used_doing_gc; static ImmortalString get_referrers_name; static PythonAllocator<ThreadState> allocator; G_NO_COPIES_OF_CLS(ThreadState); public: static void* operator new(size_t UNUSED(count)) { return ThreadState::allocator.allocate(1); } static void operator delete(void* ptr) { return ThreadState::allocator.deallocate(static_cast<ThreadState*>(ptr), 1); } static void init() { ThreadState::get_referrers_name = "get_referrers"; ThreadState::_clocks_used_doing_gc = 0; } ThreadState() : main_greenlet(OwnedMainGreenlet::consuming(green_create_main(this))), current_greenlet(main_greenlet) { if (!this->main_greenlet) { // We failed to create the main greenlet. That's bad. throw PyFatalError("Failed to create main greenlet"); } // The main greenlet starts with 1 refs: The returned one. We // then copied it to the current greenlet. assert(this->main_greenlet.REFCNT() == 2); #ifdef GREENLET_NEEDS_EXCEPTION_STATE_SAVED this->exception_state = slp_get_exception_state(); #endif } inline void restore_exception_state() { #ifdef GREENLET_NEEDS_EXCEPTION_STATE_SAVED // It's probably important this be inlined and only call C // functions to avoid adding an SEH frame. slp_set_exception_state(this->exception_state); #endif } inline bool has_main_greenlet() { return !!this->main_greenlet; } // Called from the ThreadStateCreator when we're in non-standard // threading mode. In that case, there is an object in the Python // thread state dictionary that points to us. The main greenlet // also traverses into us, in which case it's crucial not to // traverse back into the main greenlet. int tp_traverse(visitproc visit, void* arg, bool traverse_main=true) { if (traverse_main) { Py_VISIT(main_greenlet.borrow_o()); } if (traverse_main || current_greenlet != main_greenlet) { Py_VISIT(current_greenlet.borrow_o()); } Py_VISIT(tracefunc.borrow()); return 0; } inline BorrowedMainGreenlet borrow_main_greenlet() const { assert(this->main_greenlet); assert(this->main_greenlet.REFCNT() >= 2); return this->main_greenlet; }; inline OwnedMainGreenlet get_main_greenlet() { return this->main_greenlet; } /** * In addition to returning a new reference to the currunt * greenlet, this performs any maintenance needed. */ inline OwnedGreenlet get_current() { /* green_dealloc() cannot delete greenlets from other threads, so it stores them in the thread dict; delete them now. */ this->clear_deleteme_list(); //assert(this->current_greenlet->main_greenlet == this->main_greenlet); //assert(this->main_greenlet->main_greenlet == this->main_greenlet); return this->current_greenlet; } /** * As for non-const get_current(); */ inline BorrowedGreenlet borrow_current() { this->clear_deleteme_list(); return this->current_greenlet; } /** * Does no maintenance. */ inline OwnedGreenlet get_current() const { return this->current_greenlet; } template<typename T, refs::TypeChecker TC> inline bool is_current(const refs::PyObjectPointer<T, TC>& obj) const { return this->current_greenlet.borrow_o() == obj.borrow_o(); } inline void set_current(const OwnedGreenlet& target) { this->current_greenlet = target; } private: /** * Deref and remove the greenlets from the deleteme list. Must be * holding the GIL. * * If *murder* is true, then we must be called from a different * thread than the one that these greenlets were running in. * In that case, if the greenlet was actually running, we destroy * the frame reference and otherwise make it appear dead before * proceeding; otherwise, we would try (and fail) to raise an * exception in it and wind up right back in this list. */ inline void clear_deleteme_list(const bool murder=false) { if (!this->deleteme.empty()) { // It's possible we could add items to this list while // running Python code if there's a thread switch, so we // need to defensively copy it before that can happen. deleteme_t copy = this->deleteme; this->deleteme.clear(); // in case things come back on the list for(deleteme_t::iterator it = copy.begin(), end = copy.end(); it != end; ++it ) { PyGreenlet* to_del = *it; if (murder) { // Force each greenlet to appear dead; we can't raise an // exception into it anymore anyway. to_del->pimpl->murder_in_place(); } // The only reference to these greenlets should be in // this list, decreffing them should let them be // deleted again, triggering calls to green_dealloc() // in the correct thread (if we're not murdering). // This may run arbitrary Python code and switch // threads or greenlets! Py_DECREF(to_del); if (PyErr_Occurred()) { PyErr_WriteUnraisable(nullptr); PyErr_Clear(); } } } } public: /** * Returns a new reference, or a false object. */ inline OwnedObject get_tracefunc() const { return tracefunc; }; inline void set_tracefunc(BorrowedObject tracefunc) { assert(tracefunc); if (tracefunc == BorrowedObject(Py_None)) { this->tracefunc.CLEAR(); } else { this->tracefunc = tracefunc; } } /** * Given a reference to a greenlet that some other thread * attempted to delete (has a refcount of 0) store it for later * deletion when the thread this state belongs to is current. */ inline void delete_when_thread_running(PyGreenlet* to_del) { Py_INCREF(to_del); this->deleteme.push_back(to_del); } /** * Set to std::clock_t(-1) to disable. */ inline static std::clock_t& clocks_used_doing_gc() { return ThreadState::_clocks_used_doing_gc; } ~ThreadState() { if (!PyInterpreterState_Head()) { // We shouldn't get here (our callers protect us) // but if we do, all we can do is bail early. return; } // We should not have an "origin" greenlet; that only exists // for the temporary time during a switch, which should not // be in progress as the thread dies. //assert(!this->switching_state.origin); this->tracefunc.CLEAR(); // Forcibly GC as much as we can. this->clear_deleteme_list(true); // The pending call did this. assert(this->main_greenlet->thread_state() == nullptr); // If the main greenlet is the current greenlet, // then we "fell off the end" and the thread died. // It's possible that there is some other greenlet that // switched to us, leaving a reference to the main greenlet // on the stack, somewhere uncollectible. Try to detect that. if (this->current_greenlet == this->main_greenlet && this->current_greenlet) { assert(this->current_greenlet->is_currently_running_in_some_thread()); // Drop one reference we hold. this->current_greenlet.CLEAR(); assert(!this->current_greenlet); // Only our reference to the main greenlet should be left, // But hold onto the pointer in case we need to do extra cleanup. PyGreenlet* old_main_greenlet = this->main_greenlet.borrow(); Py_ssize_t cnt = this->main_greenlet.REFCNT(); this->main_greenlet.CLEAR(); if (ThreadState::_clocks_used_doing_gc != std::clock_t(-1) && cnt == 2 && Py_REFCNT(old_main_greenlet) == 1) { // Highly likely that the reference is somewhere on // the stack, not reachable by GC. Verify. // XXX: This is O(n) in the total number of objects. // TODO: Add a way to disable this at runtime, and // another way to report on it. std::clock_t begin = std::clock(); NewReference gc(PyImport_ImportModule("gc")); if (gc) { OwnedObject get_referrers = gc.PyRequireAttr(ThreadState::get_referrers_name); OwnedList refs(get_referrers.PyCall(old_main_greenlet)); if (refs && refs.empty()) { assert(refs.REFCNT() == 1); // We found nothing! So we left a dangling // reference: Probably the last thing some // other greenlet did was call // 'getcurrent().parent.switch()' to switch // back to us. Clean it up. This will be the // case on CPython 3.7 and newer, as they use // an internal calling conversion that avoids // creating method objects and storing them on // the stack. Py_DECREF(old_main_greenlet); } else if (refs && refs.size() == 1 && PyCFunction_Check(refs.at(0)) && Py_REFCNT(refs.at(0)) == 2) { assert(refs.REFCNT() == 1); // Ok, we found a C method that refers to the // main greenlet, and its only referenced // twice, once in the list we just created, // once from...somewhere else. If we can't // find where else, then this is a leak. // This happens in older versions of CPython // that create a bound method object somewhere // on the stack that we'll never get back to. if (PyCFunction_GetFunction(refs.at(0).borrow()) == (PyCFunction)green_switch) { BorrowedObject function_w = refs.at(0); refs.clear(); // destroy the reference // from the list. // back to one reference. Can *it* be // found? assert(function_w.REFCNT() == 1); refs = get_referrers.PyCall(function_w); if (refs && refs.empty()) { // Nope, it can't be found so it won't // ever be GC'd. Drop it. Py_CLEAR(function_w); } } } std::clock_t end = std::clock(); ThreadState::_clocks_used_doing_gc += (end - begin); } } } // We need to make sure this greenlet appears to be dead, // because otherwise deallocing it would fail to raise an // exception in it (the thread is dead) and put it back in our // deleteme list. if (this->current_greenlet) { this->current_greenlet->murder_in_place(); this->current_greenlet.CLEAR(); } if (this->main_greenlet) { // Couldn't have been the main greenlet that was running // when the thread exited (because we already cleared this // pointer if it was). This shouldn't be possible? // If the main greenlet was current when the thread died (it // should be, right?) then we cleared its self pointer above // when we cleared the current greenlet's main greenlet pointer. // assert(this->main_greenlet->main_greenlet == this->main_greenlet // || !this->main_greenlet->main_greenlet); // // self reference, probably gone // this->main_greenlet->main_greenlet.CLEAR(); // This will actually go away when the ivar is destructed. this->main_greenlet.CLEAR(); } if (PyErr_Occurred()) { PyErr_WriteUnraisable(NULL); PyErr_Clear(); } } }; ImmortalString ThreadState::get_referrers_name(nullptr); PythonAllocator<ThreadState> ThreadState::allocator; std::clock_t ThreadState::_clocks_used_doing_gc(0); template<typename Destructor> class ThreadStateCreator { private: // Initialized to 1, and, if still 1, created on access. // Set to 0 on destruction. ThreadState* _state; G_NO_COPIES_OF_CLS(ThreadStateCreator); public: // Only one of these, auto created per thread ThreadStateCreator() : _state((ThreadState*)1) { } ~ThreadStateCreator() { ThreadState* tmp = this->_state; this->_state = nullptr; if (tmp && tmp != (ThreadState*)1) { Destructor x(tmp); } } inline ThreadState& state() { // The main greenlet will own this pointer when it is created, // which will be right after this. The plan is to give every // greenlet a pointer to the main greenlet for the thread it // runs in; if we are doing something cross-thread, we need to // access the pointer from the main greenlet. Deleting the // thread, and hence the thread-local storage, will delete the // state pointer in the main greenlet. if (this->_state == (ThreadState*)1) { // XXX: Assuming allocation never fails this->_state = new ThreadState; // For non-standard threading, we need to store an object // in the Python thread state dictionary so that it can be // DECREF'd when the thread ends (ideally; the dict could // last longer) and clean this object up. } if (!this->_state) { throw std::runtime_error("Accessing state after destruction."); } return *this->_state; } operator ThreadState&() { return this->state(); } operator ThreadState*() { return &this->state(); } inline int tp_traverse(visitproc visit, void* arg) { if (this->_state) { return this->_state->tp_traverse(visit, arg); } return 0; } }; #if G_USE_STANDARD_THREADING == 1 // We can't use the PythonAllocator for this, because we push to it // from the thread state destructor, which doesn't have the GIL, // and Python's allocators can only be called with the GIL. typedef std::vector<ThreadState*> cleanup_queue_t; #else class cleanup_queue_t { public: inline ssize_t size() const { return 0; }; inline bool empty() const { return true; }; inline void pop_back() { throw std::out_of_range("empty queue."); }; inline ThreadState* back() { throw std::out_of_range("empty queue."); }; inline void push_back(ThreadState* g) { throw std::out_of_range("empty queue."); }; }; #endif }; // namespace greenlet #endif