Parallel run and write in python
Do a parallel imap_unordered with numProc processes
- func should always return something
- Tasks should be an iterable which items are acceptable args for func
- The output of func is loggedgradually in outFileName
- If bar is true, show a progress bar
- If errFileName is not None, all exceptions produced by func are intercepted and gradually logged in errFileName
- numProc sets the number of process to use
- if ordered is true, results are outputed in the same order as tasks
- chunksize : chunkszise for imap
- if debug is set to true, errors are also printed to stderr
Install
pip
pip3 install git+https://gitlab.tetras-libre.fr/tetras-libre/pyParallelRunAndWrite@master#egg=pyParallelRunAndWriteFor developpement
git clone https://gitlab.tetras-libre.fr/tetras-libre/pyParallelRunAndWrite
pip3 install -e .As a dependency in your setup.py
from setuptools.command.install import install
from setuptools.command.develop import develop
from setuptools import setup, find_packages
import pip
...
gitDeps = [
'git+https://gitlab.tetras-libre.fr/tetras-libre/pyParallelRunAndWrite@master#egg=pyParallelRunAndWrite',
]
def customRun(command_subclass):
orig_run = command_subclass.run
def modified_run(self):
for dep in gitDeps:
pip.main(['install', dep])
orig_run(self)
command_subclass.run = modified_run
return command_subclass
@customRun
class actionsOnInstall(install):
pass
@customRun
class actionsOnDevelop(develop):
pass
...
setup(
...
cmdclass={'install': actionsOnInstall,
'develop': actionsOnDevelop,
...
)
Usage
from parallelRunAndWrite import parallelRunAndWrite as prw
# The following dependencies are only useful for testFunc
import os
from time import sleep
from random import randint
def testFunc(arg):
ret=str(arg)+str(os.getpid())
sleep(randint(2,5))
if(randint(0,100)%2 ==0):
raise Exception("COUCOU"+ret)
return ret
prw.parallelRunAndWrite(testFunc, range(5), "testplop", numProc=2, errFileName="errors")