Skip to content
Snippets Groups Projects
Verified Commit 8526d3e6 authored by David Beniamine's avatar David Beniamine
Browse files

Small refactor

parent bbf400ce
Branches
No related tags found
No related merge requests found
......@@ -46,20 +46,14 @@ def parallelRunAndWrite(func, tasks, outFileName, bar=True, errFileName=None, nu
with mp.Pool(processes=numProc) as pool:
# Define imap function
if(ordered):
imap_func=pool.imap
else:
imap_func=pool.imap_unordered
imap_func = pool.imap_unordered if(not ordered) else pool.imap
with ExitStack() as stack:
# open files
out = stack.enter_context(open(outFileName, 'w+'))
if(errFileName):
err=stack.enter_context(open(errFileName, 'w+'))
numErrors=0
else:
err=None
err=stack.enter_context(open(errFileName, 'w+')) if(errFileName) else None
numErrors=0
start = time()
# actually do stuff
for result in imap_func(worker, [(func,t,err is not None) for t in tasks], chunksize):
......@@ -77,10 +71,7 @@ def parallelRunAndWrite(func, tasks, outFileName, bar=True, errFileName=None, nu
progressBar.update()
# Report status
if(err):
errmsg= '{} errors / {} tasks written to file "{}" \n'.format(numErrors, len(tasks), errFileName)
else:
errmsg=''
errmsg= '{} errors / {} tasks written to file "{}" \n'.format(numErrors, len(tasks), errFileName) if(errFileName) else ''
print('Output written to file "{}"\n{}Time elapsed: {}s'.format(outFileName, errmsg, int(time() - start)))
# wrapper to run user function
......@@ -91,23 +82,28 @@ def worker(args):
except:
if(handleErrors):
return (False,traceback.format_exc())
else:
raise
# Testing
def testFunc(arg):
ret=str(arg)+str(os.getpid())
sleep(randint(2,5))
if(randint(0,100)%2 ==0):
raise Exception("COUCOU"+ret)
return testFunc2(arg)
def testFunc2(arg):
ret=str(arg)+str(os.getpid())
sleep(randint(0,3))
return ret
def test():
print("Error file + debug")
parallelRunAndWrite(testFunc, range(10), "testplop", numProc=2, errFileName="errors", debug=True)
print("Error file")
parallelRunAndWrite(testFunc, range(10), "testplop", numProc=2, errFileName="errors")
print("No error file/ no rror")
parallelRunAndWrite(testFunc2, range(10), "testplop", numProc=2)
print("No error file")
parallelRunAndWrite(testFunc, range(10), "testplop", numProc=2)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment