diff --git a/parallelRunAndWrite.py b/parallelRunAndWrite.py index 837af07744ea062e976023bc1f06ddfb5434d65f..7601e8bcaec43f5898c175d7f04665510505b02b 100644 --- a/parallelRunAndWrite.py +++ b/parallelRunAndWrite.py @@ -46,20 +46,14 @@ def parallelRunAndWrite(func, tasks, outFileName, bar=True, errFileName=None, nu with mp.Pool(processes=numProc) as pool: # Define imap function - if(ordered): - imap_func=pool.imap - else: - imap_func=pool.imap_unordered + imap_func = pool.imap_unordered if(not ordered) else pool.imap with ExitStack() as stack: # open files out = stack.enter_context(open(outFileName, 'w+')) - if(errFileName): - err=stack.enter_context(open(errFileName, 'w+')) - numErrors=0 - else: - err=None + err=stack.enter_context(open(errFileName, 'w+')) if(errFileName) else None + numErrors=0 start = time() # actually do stuff for result in imap_func(worker, [(func,t,err is not None) for t in tasks], chunksize): @@ -77,10 +71,7 @@ def parallelRunAndWrite(func, tasks, outFileName, bar=True, errFileName=None, nu progressBar.update() # Report status - if(err): - errmsg= '{} errors / {} tasks written to file "{}" \n'.format(numErrors, len(tasks), errFileName) - else: - errmsg='' + errmsg= '{} errors / {} tasks written to file "{}" \n'.format(numErrors, len(tasks), errFileName) if(errFileName) else '' print('Output written to file "{}"\n{}Time elapsed: {}s'.format(outFileName, errmsg, int(time() - start))) # wrapper to run user function @@ -91,23 +82,28 @@ def worker(args): except: if(handleErrors): return (False,traceback.format_exc()) - else: - raise + raise # Testing def testFunc(arg): - ret=str(arg)+str(os.getpid()) - sleep(randint(2,5)) if(randint(0,100)%2 ==0): raise Exception("COUCOU"+ret) + return testFunc2(arg) + +def testFunc2(arg): + ret=str(arg)+str(os.getpid()) + sleep(randint(0,3)) return ret + def test(): print("Error file + debug") parallelRunAndWrite(testFunc, range(10), "testplop", numProc=2, errFileName="errors", debug=True) print("Error file") parallelRunAndWrite(testFunc, range(10), "testplop", numProc=2, errFileName="errors") + print("No error file/ no rror") + parallelRunAndWrite(testFunc2, range(10), "testplop", numProc=2) print("No error file") parallelRunAndWrite(testFunc, range(10), "testplop", numProc=2)