Skip to content
Snippets Groups Projects
Commit f81ae68f authored by Aurélien Lamercerie's avatar Aurélien Lamercerie
Browse files

Update CTS loader function (in tenet extraction)

parent e1c9b53c
Branches
Tags
No related merge requests found
......@@ -19,6 +19,8 @@ import logging
import glob
from pathlib import Path
from importlib.machinery import SourceFileLoader
import importlib.util
import importlib
from .timer import timed
from .transduction.rule import Rule
from .transduction.sequence import Sequence
......@@ -81,11 +83,31 @@ def load_rule_set(config, rule_dir, prefix_list):
try:
rule_set = {}
target_ref = config.cts_dir + rule_dir + '**/*.py'
for rule_filename in glob.iglob(target_ref, recursive = True):
# -- Append rule paths to sys.path
path_glob_pattern = config.cts_dir + rule_dir + '**/'
for rule_path in glob.iglob(path_glob_pattern, recursive = True):
sys.path.append(rule_path)
# -- Load rule modules
file_glob_pattern = config.cts_dir + rule_dir + '**/*.py'
for rule_filename in glob.iglob(file_glob_pattern, recursive = True):
if 'query_builder' not in rule_filename:
# -- old ---
# spec = importlib.util.spec_from_file_location(rule_module_name,
# rule_filename)
# rule_module = importlib.util.module_from_spec(spec)
# sys.modules[rule_module_name] = rule_module
# spec.loader.exec_module(rule_module)
# -- ---
# -- Import module
rule_module_name = Path(rule_filename).stem
rule_loader = SourceFileLoader(rule_module_name, rule_filename)
rule_module = rule_loader.load_module()
rule_module = importlib.import_module(f'{rule_module_name}')
# -- Update rule set
if hasattr(rule_module, 'rule_set'):
rule_def_set = rule_module.rule_set
new_rule_set = get_new_rule_set(rule_def_set, prefix_list)
rule_set.update(new_rule_set)
......@@ -94,14 +116,15 @@ def load_rule_set(config, rule_dir, prefix_list):
except:
logger.error(' *** Error while loading rule set (load_rule_set) *** ')
logger.debug(' ----- target_ref: {0}'.format(target_ref))
logger.debug(' ----- rule_filename: {0}'.format(rule_filename))
logger.debug(' ----- rule_module_name: {0}'.format(rule_module_name))
logger.debug(' ----- rule_loader: {0}'.format(rule_loader))
logger.debug(' ----- rule_module: {0}'.format(rule_module))
logger.debug(' ----- len(rule_def_set): {0}'.format(len(rule_def_set)))
logger.debug(' ----- len(new_rule_set): {0}'.format(len(new_rule_set)))
logger.debug(' ----- len(rule_set): {0}'.format(len(rule_set)))
logger.debug(f' ----- path_glob_pattern: {path_glob_pattern}')
logger.debug(f' ----- sys.path: {sys.path}')
logger.debug(f' ----- file_glob_pattern: {file_glob_pattern}')
logger.debug(f' ----- rule_filename: {rule_filename}')
logger.debug(f' ----- rule_module_name: {rule_module_name}')
logger.debug(f' ----- rule_module: {rule_module}')
logger.debug(f' ----- len(rule_def_set): {len(rule_def_set)}')
logger.debug(f' ----- len(new_rule_set): {len(new_rule_set)}')
logger.debug(f' ----- len(rule_set): {len(rule_set)}')
......
......@@ -11,7 +11,7 @@
# Importing required modules
#==============================================================================
# --
#==============================================================================
......
......@@ -23,8 +23,8 @@
----- CTS directory: ./structure/cts/
----- target frame directory: ./input/targetFrameStructure/
----- input document directory: ./input/amrDocuments/
----- output directory: ./output/SolarSystemDev1-20221212/
----- sentence output directory: ./output/SolarSystemDev1-20221212/
----- output directory: ./output/SolarSystemDev1-20221213/
----- sentence output directory: ./output/SolarSystemDev1-20221213/
----- SHACL binary directory: ./lib/shacl-1.3.2/bin
-- Config File Definition
----- schema file: ./structure/amr-rdf-schema.ttl
......@@ -44,9 +44,9 @@
----- frame ontology seed file: ./input/targetFrameStructure/base-ontology-seed.ttl
-- Output
----- ontology namespace: https://tenet.tetras-libre.fr/base-ontology/
----- output file: ./output/SolarSystemDev1-20221212/SolarSystemDev1.ttl
----- output file: ./output/SolarSystemDev1-20221213/SolarSystemDev1.ttl
*** - ***
- INFO - -- Creating output target directory: ./output/SolarSystemDev1-20221212/
- INFO - -- Creating output target directory: ./output/SolarSystemDev1-20221213/
- DEBUG - -- Counting number of graph files (sentences)
- DEBUG - ----- Graph count: 1
- INFO - === Extraction Processing using New TENET Engine ===
......@@ -64,9 +64,9 @@
- DEBUG - ----- Sentence Loading
- DEBUG - -------- ./input/amrDocuments/dev/solar-system-1/SSC-01-01.stog.amr.ttl (614)
- DEBUG - --- Export work graph as turtle
- DEBUG - ----- Work graph file: ./output/SolarSystemDev1-20221212/SolarSystemDev1-1/SolarSystemDev1.ttl
- DEBUG - ----- Work graph file: ./output/SolarSystemDev1-20221213/SolarSystemDev1-1/SolarSystemDev1.ttl
- DEBUG - --- Ending Structure Preparation
- DEBUG - ----- Total Execution Time = 0:00:00.155235
- DEBUG - ----- Total Execution Time = 0:00:00.132065
- INFO - -- Loading Extraction Scheme (amr_scheme_1)
- DEBUG - ----- Step number: 3
- INFO - -- Loading Extraction Rules (amr_ctr/*)
......@@ -95,7 +95,7 @@
- DEBUG - --- Serializing graph to SolarSystemDev1_preprocessing
- DEBUG - ----- step: preprocessing
- DEBUG - ----- id: SolarSystemDev1
- DEBUG - ----- work_file: ./output/SolarSystemDev1-20221212/SolarSystemDev1-1/SolarSystemDev1_preprocessing.ttl
- DEBUG - ----- work_file: ./output/SolarSystemDev1-20221213/SolarSystemDev1-1/SolarSystemDev1_preprocessing.ttl
- DEBUG - ----- base: http://SolarSystemDev1/preprocessing
- INFO - ----- 212 triples extracted during preprocessing step
- INFO - -- Applying extraction step: transduction
......@@ -176,7 +176,7 @@
- DEBUG - --- Serializing graph to SolarSystemDev1_transduction
- DEBUG - ----- step: transduction
- DEBUG - ----- id: SolarSystemDev1
- DEBUG - ----- work_file: ./output/SolarSystemDev1-20221212/SolarSystemDev1-1/SolarSystemDev1_transduction.ttl
- DEBUG - ----- work_file: ./output/SolarSystemDev1-20221213/SolarSystemDev1-1/SolarSystemDev1_transduction.ttl
- DEBUG - ----- base: http://SolarSystemDev1/transduction
- INFO - ----- 376 triples extracted during transduction step
- INFO - -- Applying extraction step: generation
......@@ -205,23 +205,23 @@
- DEBUG - --- Serializing graph to SolarSystemDev1_generation
- DEBUG - ----- step: generation
- DEBUG - ----- id: SolarSystemDev1
- DEBUG - ----- work_file: ./output/SolarSystemDev1-20221212/SolarSystemDev1-1/SolarSystemDev1_generation.ttl
- DEBUG - ----- work_file: ./output/SolarSystemDev1-20221213/SolarSystemDev1-1/SolarSystemDev1_generation.ttl
- DEBUG - ----- base: http://SolarSystemDev1/generation
- INFO - ----- 159 triples extracted during generation step
- INFO - -- Result: file containing only the factoids
- DEBUG - --- Making factoid graph with the last step result
- DEBUG - ----- Number of factoids: 176
- DEBUG - ----- Graph base: http://SolarSystemDev1/factoid
- DEBUG - --- Serializing graph to factoid file (./output/SolarSystemDev1-20221212/SolarSystemDev1-1/SolarSystemDev1_factoid.ttl)
- DEBUG - --- Serializing graph to factoid file (./output/SolarSystemDev1-20221213/SolarSystemDev1-1/SolarSystemDev1_factoid.ttl)
- INFO -
*** Execution Time ***
----- Function: apply (lib.tenet_extraction)
----- Total Time: 0:00:09.171347
----- Process Time: 0:00:09.035147
----- Total Time: 0:00:07.356953
----- Process Time: 0:00:07.333204
*** - ***
- INFO - === Final Ontology Generation ===
- INFO - -- Making complete factoid graph by merging sentence factoid graphs
- INFO - ----- Total factoid number: 176
- INFO - ----- Graph base: http://SolarSystemDev1/factoid
- INFO - -- Serializing graph to factoid file (./output/SolarSystemDev1-20221212/SolarSystemDev1_factoid.ttl)
- INFO - -- Serializing graph to factoid file (./output/SolarSystemDev1-20221213/SolarSystemDev1_factoid.ttl)
- INFO - === Done ===
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment