Skip to content
Snippets Groups Projects
Commit 2224211a authored by David Rouquet's avatar David Rouquet
Browse files

Multiprocessing run with initializer

parent af8a5cee
No related branches found
No related tags found
No related merge requests found
......@@ -16,6 +16,7 @@ import traceback
import logging
import multiprocessing_logging
import multiprocessing
from multiprocessing import Manager
from amrlib.graph_processing.amr_plot import AMRPlot
from filepath_manager import FilepathManager
......@@ -89,21 +90,26 @@ def __generate_sentence_file(filepath_manager, workdata_list):
first = False
#==============================================================================
# Sentence Conversion to AMR
#==============================================================================
# Function executed when a worker is created in the pool
def init_pool_worker():
# declare scope of a new global variable
global stog
amr_model_path = "/home/daxid/hdd_data/jupyterlab_root/lib/amrModel/model_parse_xfm_bart_large-v0_1_0"
# store argument in the global variable for this process
logger.info("-- Loading AMR model")
stog = amrlib.load_stog_model(model_dir=amr_model_path)
def __run_conversion(arg_dict):
data = arg_dict['data']
amr_model_path = arg_dict['amr_model_path']
logger.info("-- Loading AMR model")
stog = amrlib.load_stog_model(model_dir=amr_model_path)
# logger.info("-- Loading AMR model")
# stog = amrlib.load_stog_model(model_dir=amr_model_path)
logger.info("-- Converting sentences to AMR graphs")
stog_result = stog.parse_sents([data.sentence])
......@@ -117,13 +123,16 @@ def __run_conversion(arg_dict):
def __convert_sentences_to_graphs(amr_model_path, input_data_list):
""" Converting text sentences to AMR graphs """
global stog
mapIterable = []
for data in input_data_list:
arg_dict = { 'data': data, 'amr_model_path': amr_model_path}
mapIterable = mapIterable + [arg_dict]
number_of_processes = min(round((multiprocessing.cpu_count()-1)/4), len(input_data_list))
with multiprocessing.Pool(number_of_processes) as p:
#with multiprocessing.Pool(number_of_processes) as p:
with multiprocessing.Pool(number_of_processes, initializer=init_pool_worker) as p:
result_data_list = p.map(__run_conversion, mapIterable)
# result_data_list = []
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment