Multiprocessing in Mayapy

Tools Maya
, ,
1

I can’t vouch for this solution, which I found in the Autodesk forums (from a user named “DrPangloss”) but I thought I’d post it here in case anybody is still banging their heard against the nightmare of trying to use vanilla multiprocessing code patterns with MayaPy. If you try it and it works, let us know how it goes!

def fix_multiprocessing():
    import importlib.util
    if importlib.util.find_spec("_posixshmem"):
        return

    import sys
    import os
    import tempfile
    import subprocess
    import shutil
    import glob
    import sysconfig

    python_include_dir = None
    for root, dirs, files in os.walk(sys.base_prefix):
        if 'pyconfig.h' in files and 'Python.h' in files:
            python_include_dir = root
            break
    
    build_script_source = f"""
from cffi import FFI as _FFI
import sys
_ffi = _FFI()
_ffi.cdef('''
typedef int... mode_t;
int shm_open(const char *name, int oflag, mode_t mode);
int shm_unlink(const char *name);
''')
SOURCE = '''
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
'''
libraries = [] if sys.platform == 'darwin' else ['rt']
include_dirs = ['{python_include_dir}'] if python_include_dir else []
_ffi.set_source(
    "_posixshmem_cffi",
    SOURCE,
    libraries=libraries,
    include_dirs=include_dirs
)
if __name__ == "__main__":
    _ffi.compile()
"""

    module_source = """
from _posixshmem_cffi import lib, ffi
import errno
import os
def shm_open(path, flags, mode=0o777):
    path_utf8 = path.encode("utf-8")
    if b'\\x00' in path_utf8:
        raise ValueError('embedded null character')
    while 1:
        fd = lib.shm_open(path_utf8, flags, mode)
        if fd < 0:
            e = ffi.errno
            if e != errno.EINTR:
                raise OSError(e, os.strerror(e))
        else:
            return fd
def shm_unlink(path):
    path_utf8 = path.encode("utf-8")
    if b'\\x00' in path_utf8:
        raise ValueError('embedded null character')
    while 1:
        rv = lib.shm_unlink(path_utf8)
        if rv < 0:
            e = ffi.errno
            if e != errno.EINTR:
                raise OSError(e, os.strerror(e))
        else:
            return
"""
    with tempfile.TemporaryDirectory() as build_dir:
        build_script_path = os.path.join(build_dir, '_posixshmem_build.py')
        module_path = os.path.join(build_dir, '_posixshmem.py')

        with open(build_script_path, 'w') as f:
            f.write(build_script_source)

        with open(module_path, 'w') as f:
            f.write(module_source)
        
        subprocess.run(
            [sys.executable, build_script_path],
            cwd=build_dir,
            check=True
        )

        compiled_so = glob.glob(os.path.join(build_dir, '_posixshmem_cffi*.so'))                    
        site_packages = sysconfig.get_paths()["purelib"]
        shutil.copy(compiled_so, site_packages)
        shutil.copy(module_path, site_packages)
    
    import importlib
    importlib.invalidate_caches()

fix_multiprocessing()
2 Likes
2

Reading through this, is it fixing how memory gets shared between processes? So I’m guessing mmap stuff? Is this required if you’re only using socket connections instead of an mmap?

And since it’s got “posix” in the library name and the extension is “.so”, I’m assuming this is only for mac and linux?

3

Yeah this looks like a *nix option, though I guess if you could compile the same stuff to a dll in windows it’d work there too… maybe?

1 Like