can't get lru_cache decorated function

[basnijholt]

This appears in python-adaptive/adaptive#292.

from functools import lru_cache
from loky.reusable_executor import get_reusable_executor

@lru_cache
def g(x):
    return x

def f(x):
    return g(x)

with get_reusable_executor() as ex:
    fut = ex.submit(f, 1)
    print(fut.result())

fails with

---------------------------------------------------------------------------
_RemoteTraceback                          Traceback (most recent call last)
_RemoteTraceback: 
"""
Traceback (most recent call last):
  File "/gscratch/home/a-banijh/miniconda3/envs/py38-new-adaptive-scheduler/lib/python3.8/site-packages/loky/process_executor.py", line 404, in _process_worker
    call_item = call_queue.get(block=True, timeout=timeout)
  File "/gscratch/home/a-banijh/miniconda3/envs/py38-new-adaptive-scheduler/lib/python3.8/multiprocessing/queues.py", line 116, in get
    return _ForkingPickler.loads(res)
AttributeError: Can't get attribute 'g' on <module 'loky.backend.popen_loky_posix' from '/gscratch/home/a-banijh/miniconda3/envs/py38-new-adaptive-scheduler/lib/python3.8/site-packages/loky/backend/popen_loky_posix.py'>
"""

The above exception was the direct cause of the following exception:

BrokenProcessPool                         Traceback (most recent call last)
<ipython-input-6-7b5a323a86c8> in <module>
     12 with get_reusable_executor() as ex:
     13     fut = ex.submit(f, 1)
---> 14     print(fut.result())

~/miniconda3/envs/py38-new-adaptive-scheduler/lib/python3.8/concurrent/futures/_base.py in result(self, timeout)
    437                 raise CancelledError()
    438             elif self._state == FINISHED:
--> 439                 return self.__get_result()
    440             else:
    441                 raise TimeoutError()

~/miniconda3/envs/py38-new-adaptive-scheduler/lib/python3.8/concurrent/futures/_base.py in __get_result(self)
    386     def __get_result(self):
    387         if self._exception:
--> 388             raise self._exception
    389         else:
    390             return self._result

BrokenProcessPool: A task has failed to un-serialize. Please ensure that the arguments of the function are all picklable.

On the other hand, the concurrent.futures.ProcessPoolExecutor just works fine in this case.

[basnijholt]

This is because of cloudpipe/cloudpickle#178
and can be reproduced with

import cloudpickle
from functools import lru_cache

@lru_cache
def g(x):
    return x

dump = cloudpickle.dumps(g)
del g
g = cloudpickle.loads(dump)
g(1)