diff --git a/tenet/main.py b/tenet/main.py index 2346b398703a4812f926a13945548fdfe193277a..9d2fcbe5ac6621ef9fdfecd93d80beabbb415e62 100644 --- a/tenet/main.py +++ b/tenet/main.py @@ -13,7 +13,7 @@ from rdflib import Graph import logging import logging.config import multiprocessing_logging -from multiprocessing import Pool +import multiprocessing from extraction import config, structure, process from utility.timer import timed @@ -166,6 +166,16 @@ def create_ontology_from_amrld_file(amrld_file_path, return ontology_turtle_string +global result_triple_queue + +def pool_function(sentence_indice,sentence_file_list): + sentence_file = sentence_file_list[sentence_indice] + logger.info(f' *** sentence {sentence_indice} *** ') + config.sentence_output_dir = f'-{sentence_indice}' + new_triple_list = __apply_extraction(config, sentence_file) + # The following must handled via a global queue + result_triple_list.extend(new_triple_list) + @timed def create_ontology_from_amrld_dir(amrld_dir_path, @@ -173,7 +183,7 @@ def create_ontology_from_amrld_dir(amrld_dir_path, onto_prefix=None, out_file_path=None, technical_dir_path=None, - processes=1 + processes=multiprocessing.cpu_count()-1 ): """ Method to create an ontology (as Turtle String) from a transduction @@ -212,13 +222,17 @@ def create_ontology_from_amrld_dir(amrld_dir_path, sentence_dir = config.source_sentence_file sentence_count = 0 result_triple_list = [] -#@@ Support for multiprecessing mus be done here - for sentence_file in glob.glob(sentence_dir, recursive = True): - sentence_count += 1 - logger.info(f' *** sentence {sentence_count} *** ') - config.sentence_output_dir = f'-{sentence_count}' - new_triple_list = __apply_extraction(config, sentence_file) - result_triple_list.extend(new_triple_list) + + result_triple_queue = multiprocessing.Queue() + sentence_file_list = glob.glob(sentence_dir, recursive = True) + + # The following is for multiprocessing logging (must be exec before the pool is created + multiprocessing_logging.install_mp_handler() + + star_iterable = [(i, sentence_file_list) for i in range(len(sentence_file_list))] + + with multiprocessing.Pool(processes) as p: + p.starmap(pool_function, star_iterable) # -- Final Ontology Generation (factoid_graph) logger.info('\n === Final Ontology Generation === ')