dynamically generated proxies in a submodule can't be serialized when used in manager being attribute of another manager

[malmiteria]

The title says it all.
I got this error message :

_pickle.PicklingError: Can't pickle <class 'helper.makeProxy.<locals>.CustomProxy'>: it's not found as helper.makeProxy.<locals>.CustomProxy

when running the code below :

main.py file :

from multiprocess.managers import BaseManager, NamespaceProxy
from helper import makeProxy


class Managed:
    pass

class Holder:
    def __init__(self, managed):
        self.managed = managed

if __name__ == "__main__":
    BaseManager.register('M', Managed, makeProxy())
    m = BaseManager()
    m.start()
    managed = m.M()

    BaseManager.register('H', Holder, NamespaceProxy)
    m = BaseManager()
    m.start()
    holder = m.H(managed)

helper.py file :

from multiprocess.managers import NamespaceProxy

def makeProxy():
    class CustomProxy(NamespaceProxy):
        pass
    return CustomProxy

I tried different things, and this seems to be the minimal bug case.

• I tried placing the makeProxy method in the main module, and the bug doesn't appear

main.py file :

from multiprocess.managers import BaseManager, NamespaceProxy

class Managed:
    pass

class Holder:
    def __init__(self, managed):
        self.managed = managed

def makeProxy():
    class CustomProxy(NamespaceProxy):
        pass
    return CustomProxy

if __name__ == "__main__":
    BaseManager.register('M', Managed, makeProxy())
    m = BaseManager()
    m.start()
    managed = m.M()
    BaseManager.register('H', Holder, NamespaceProxy)
    m = BaseManager()
    m.start()
    holder = m.H(managed)

• I tried having the makeProxy method directly return NamespaceProxy, and the bug doesn't appear

helper.py file :

from multiprocess.managers import NamespaceProxy

def makeProxy():
    return NamespaceProxy

main.py file as in the bug case

• I tried not passing the Managed instance as attribute of the Holder instance, and the bug doesn't appear

main.py file :

from multiprocess.managers import BaseManager
from helper import makeProxy


class Managed:
    pass

class Holder:
    pass


if __name__ == "__main__":
    BaseManager.register('M', Managed, makeProxy())
    m = BaseManager()
    m.start()
    managed = m.M()
    BaseManager.register('H', Holder, NamespaceProxy)
    m = BaseManager()
    m.start()
    holder = m.H()

helper.py file as in the bug case

• I tried having the proxies generated in the managed instances, but not used as proxies for the registration of that manager, and the bug doesn't appear

main.py file :

from multiprocess.managers import BaseManager, NamespaceProxy
from helper import makeProxy


class Managed:
    def run(self):
        makeProxy()

class Holder:
    def __init__(self, managed):
        self.managed = managed
    def run(self):
        makeProxy()


if __name__ == "__main__":
    BaseManager.register('M', Managed, NamespaceProxy)
    m = BaseManager()
    m.start()
    managed = m.M()
    managed.run()
    BaseManager.register('H', Holder, NamespaceProxy)
    m = BaseManager()
    m.start()
    holder = m.H(managed)
    holder.run()

helper.py file as in the bug case.

seems to work with any classes inheriting from any proxies available in the managers module, I mainly tested it on NamespaceProxy and ListProxy

[malmiteria]

I recently found a simpler version of this bug, using the MakeProxyType from the manager module.
Same bug, but it involves less custom code:

from multiprocess.managers import BaseManager, NamespaceProxy, MakeProxyType


class Managed:
    pass

class Holder:
    def __init__(self, managed):
        self.managed = managed


if __name__ == "__main__":
    BaseManager.register('M', Managed, MakeProxyType('M', ()))
    m = BaseManager()
    m.start()
    managed = m.M()

    BaseManager.register('H', Holder, NamespaceProxy)
    m = BaseManager()
    m.start()
    holder = m.H(managed)

If you're curious about a full traceback, here it is :

Traceback (most recent call last):
  File "main.py", line 21, in <module>
    holder = m.H(managed)
  File "/home/martin/src/chips/local.virtualenv/lib/python3.7/site-packages/multiprocess/managers.py", line 724, in temp
    token, exp = self._create(typeid, *args, **kwds)
  File "/home/martin/src/chips/local.virtualenv/lib/python3.7/site-packages/multiprocess/managers.py", line 609, in _create
    id, exposed = dispatch(conn, None, 'create', (typeid,)+args, kwds)
  File "/home/martin/src/chips/local.virtualenv/lib/python3.7/site-packages/multiprocess/managers.py", line 78, in dispatch
    c.send((id, methodname, args, kwds))
  File "/home/martin/src/chips/local.virtualenv/lib/python3.7/site-packages/multiprocess/connection.py", line 209, in send
    self._send_bytes(_ForkingPickler.dumps(obj))
  File "/home/martin/src/chips/local.virtualenv/lib/python3.7/site-packages/multiprocess/reduction.py", line 54, in dumps
    cls(buf, protocol, *args, **kwds).dump(obj)
  File "/home/martin/src/chips/local.virtualenv/lib/python3.7/site-packages/dill/_dill.py", line 498, in dump
    StockPickler.dump(self, obj)
  File "/usr/lib/python3.7/pickle.py", line 437, in dump
    self.save(obj)
  File "/usr/lib/python3.7/pickle.py", line 504, in save
    f(self, obj) # Call unbound method with explicit self
  File "/usr/lib/python3.7/pickle.py", line 789, in save_tuple
    save(element)
  File "/usr/lib/python3.7/pickle.py", line 504, in save
    f(self, obj) # Call unbound method with explicit self
  File "/usr/lib/python3.7/pickle.py", line 774, in save_tuple
    save(element)
  File "/usr/lib/python3.7/pickle.py", line 549, in save
    self.save_reduce(obj=obj, *rv)
  File "/usr/lib/python3.7/pickle.py", line 638, in save_reduce
    save(args)
  File "/usr/lib/python3.7/pickle.py", line 504, in save
    f(self, obj) # Call unbound method with explicit self
  File "/usr/lib/python3.7/pickle.py", line 789, in save_tuple
    save(element)
  File "/usr/lib/python3.7/pickle.py", line 504, in save
    f(self, obj) # Call unbound method with explicit self
  File "/home/martin/src/chips/local.virtualenv/lib/python3.7/site-packages/dill/_dill.py", line 1439, in save_type
    StockPickler.save_global(pickler, obj, name=name)
  File "/usr/lib/python3.7/pickle.py", line 960, in save_global
    (obj, module_name, name)) from None
_pickle.PicklingError: Can't pickle <class 'multiprocess.managers.n'>: it's not found as multiprocess.managers.M