Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
amrBatch
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Tetras MARS
amrBatch
Commits
5bd63f5b
Commit
5bd63f5b
authored
2 years ago
by
David Rouquet
Browse files
Options
Downloads
Patches
Plain Diff
not working state with init workers
parent
d58918a3
Branches
Branches containing commit
No related tags found
No related merge requests found
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
amrbatch/main.py
+46
-27
46 additions, 27 deletions
amrbatch/main.py
with
46 additions
and
27 deletions
amrbatch/main.py
+
46
−
27
View file @
5bd63f5b
...
...
@@ -13,7 +13,9 @@ import subprocess
import
amrlib
from
rdflib
import
Graph
import
traceback
import
logging.config
import
logging
import
multiprocessing_logging
import
multiprocessing
from
amrlib.graph_processing.amr_plot
import
AMRPlot
from
filepath_manager
import
FilepathManager
...
...
@@ -92,17 +94,16 @@ def __generate_sentence_file(filepath_manager, workdata_list):
def
__generate_penman_amr_graph
(
filepath_manager
,
data
):
"""
AMR graph generation in penman format
"""
output_filepath
=
data
.
get_penman_amr_graph_output_filepath
()
logger
.
debug
(
f
"
----- AMR Graph file (penman):
{
os
.
path
.
basename
(
output_filepath
)
}
"
)
with
open
(
output_filepath
,
"
w
"
)
as
writing_file
:
# w = write
writing_file
.
write
(
data
.
id_line_str
)
writing_file
.
write
(
data
.
graph
)
return
(
output_filepath
)
def
__generate_dot_amr_graph
(
filepath_manager
,
data
):
"""
AMR graph generation in dot and png format
"""
try
:
# -- generating dot/png/svg files using AMRLib and GraphViz
dot_filename
=
data
.
get_dot_amr_graph_output_filepath
()
...
...
@@ -115,7 +116,7 @@ def __generate_dot_amr_graph(filepath_manager, data):
good_png_fn
=
data
.
get_png_amr_graph_output_filepath
()
logger
.
debug
(
f
'
----- AMR Graph file (png): {{os.path.basename(good_png_fn)}}
'
)
os
.
rename
(
render_fn
,
good_png_fn
)
returnValue
=
dot_filename
format
=
'
svg
'
plot
=
AMRPlot
(
dot_filename
,
format
)
plot
.
build_from_graph
(
data
.
graph
)
...
...
@@ -124,37 +125,51 @@ def __generate_dot_amr_graph(filepath_manager, data):
good_svg_fn
=
good_png_fn
.
replace
(
'
.png
'
,
'
.svg
'
)
logger
.
debug
(
f
'
----- AMR Graph file (svg): {{os.path.basename(good_svg_fn)}}
'
)
os
.
rename
(
render_fn
,
good_svg_fn
)
except
:
logger
.
warning
(
'
Exception when trying to plot
'
)
except
Exception
as
ex
:
logger
.
warning
(
'
Exception when trying to plot:
'
+
ex
)
traceback
.
print_exc
()
def
__convert_sentences_to_graphs
(
amr_model
,
workdata_list
):
"""
Converting text sentences to AMR graphs
"""
returnValue
=
'
Exception when trying to plot
'
return
(
returnValue
)
# Function executed when a worker is created in the pool
def
init_pool_worker
():
amr_model_path
=
"
/home/daxid/hdd_data/jupyterlab_root/lib/amrModel/model_parse_xfm_bart_large-v0_1_0
"
# declare scope of a new global variable
global
stog
# store argument in the global variable for this process
logger
.
info
(
"
-- Loading AMR model
"
)
stog
=
amrlib
.
load_stog_model
(
model_dir
=
amr_model
)
logger
.
info
(
"
-- Converting sentences to AMR graphs
"
)
wd_number
=
0
for
data
in
workdata_list
:
wd_number
+=
1
def
__convert_sentence_to_graph_multiprocess_run
(
data
):
print
(
"
in worker
\n
"
)
wd_number
=
1
stog_result
=
stog
.
parse_sents
([
data
.
sentence
])
logger
.
info
(
f
'
----- Sentence
{
wd_number
}
successfully processed
'
)
logger
.
debug
(
stog_result
)
data
.
graph
=
stog_result
[
0
]
return
(
stog_result
)
logger
.
info
(
f
'
----- Total processed graph number:
{
wd_number
}
'
)
def
__convert_sentences_to_graphs
(
amr_model_path
,
workdata_list
):
"""
Converting text sentences to AMR graphs
"""
# ----- (Multi-processing) Extraction Run
number_of_processes
=
min
(
multiprocessing
.
cpu_count
()
-
1
,
len
(
workdata_list
))
global
stog
with
multiprocessing
.
Pool
(
2
,
initializer
=
init_pool_worker
)
as
p
:
logger
.
info
(
"
-- Converting sentences to AMR graphs
"
)
print
(
"
pool created
\n
"
)
stog_result_list
=
p
.
map
(
__convert_sentence_to_graph_multiprocess_run
,
workdata_list
)
logger
.
info
(
f
'
----- Total processed graph number:
{
len
(
stog_result_list
)
}
'
)
return
workdata_list
def
__generate_amr_graph_files
(
filepath_manager
,
workdata_list
):
logger
.
info
(
"
-- Generating AMR graph files
"
)
for
data
in
workdata_list
:
__generate_penman_amr_graph
(
filepath_manager
,
data
)
__generate_dot_amr_graph
(
filepath_manager
,
data
)
# ----- Prepare multiprocessing data
starmapIterable
=
[(
data
,
filepath_manager
)
for
data
in
workdata_list
]
# ----- (Multi-processing) Extraction Run
with
multiprocessing
.
Pool
(
multiprocessing
.
cpu_count
()
-
1
)
as
p
:
penmanFilePathList
=
p
.
starmap
(
__generate_penman_amr_graph
,
starmapIterable
)
dotFilePathList
=
p
.
starmap
(
__generate_dot_amr_graph
,
starmapIterable
)
...
...
@@ -237,6 +252,10 @@ def __analyze_line_set_to_produce_amr_graphs(line_set, data_reference, amr_model
logger
.
info
(
f
'
-- library: amrlib
'
)
logger
.
info
(
f
'
-- model:
{
os
.
path
.
basename
(
amr_model_path
)
}
'
)
logger
.
debug
(
f
'
(
{
amr_model_path
}
)
'
)
# ----- Multiprocessing Logging (must be exec before the pool is created)
multiprocessing_logging
.
install_mp_handler
()
workdata_list
=
__convert_sentences_to_graphs
(
amr_model_path
,
workdata_list
)
__generate_amr_graph_files
(
filepath_manager
,
workdata_list
)
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment