From af5d2ee6d7d5b4c3109bdc005531f8d8df7205f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Lamercerie?=
<aurelien.lamercerie@tetras-libre.fr>
Date: Fri, 11 Aug 2023 19:17:04 +0200
Subject: [PATCH] Update metrics computing for entities
---
main.py | 2 +-
ontoScorer/metric_score.py | 4 ++
ontoScorer/metrics.py | 90 +++++++++++++++++++++---------------
ontoScorer/ontology.py | 5 ++
ontoScorer/report.py | 95 ++++++++++++++++++++++++++++----------
ontoScorer/scorer.py | 5 +-
tests/test_metrics.py | 6 +--
7 files changed, 138 insertions(+), 69 deletions(-)
diff --git a/main.py b/main.py
index fe59030..4c53ecd 100644
--- a/main.py
+++ b/main.py
@@ -21,7 +21,7 @@ def main():
scorer = OntoScorer(REFERENCE_ONTOLOGY_PATH, GENERATED_ONTOLOGY_PATH)
# Compare the ontologies
- scorer.compare()
+ scorer.compute_metrics()
# Generate a report
scorer.generate_report()
diff --git a/ontoScorer/metric_score.py b/ontoScorer/metric_score.py
index d25555f..20027ad 100644
--- a/ontoScorer/metric_score.py
+++ b/ontoScorer/metric_score.py
@@ -25,6 +25,8 @@ class Score:
self.f1 = None
self.total_elements = 0
self.matched_elements = 0
+ self.y_true = [] # Nouveau champ
+ self.y_pred = [] # Nouveau champ
#--------------------------------------------------------------------------
@@ -42,6 +44,8 @@ class Score:
Returns:
None
"""
+ self.y_true = y_true
+ self.y_pred = y_pred
self.precision = precision_score(y_true, y_pred)
self.recall = recall_score(y_true, y_pred)
self.f1 = f1_score(y_true, y_pred)
diff --git a/ontoScorer/metrics.py b/ontoScorer/metrics.py
index b3c2b38..3c4936f 100644
--- a/ontoScorer/metrics.py
+++ b/ontoScorer/metrics.py
@@ -15,6 +15,7 @@ matches the reference.
from ontoScorer.ontology import Ontology
from ontoScorer.metric_score import Score
+
class Metrics:
"""
Metrics class provides functionalities to compute scores for ontology
@@ -26,59 +27,73 @@ class Metrics:
#--------------------------------------------------------------------------
def __init__(self):
- """
- Initializes score categories for various ontology elements.
- """
self.scores = {
- "class": Score(),
- "object_property": Score(),
- "data_property": Score(),
- "restriction": Score(),
- "individual": Score(),
- "annotation": Score(),
- "overall": Score()
+ "entities": {
+ "classes": Score(),
+ "object_properties": Score(),
+ "individuals": Score(),
+ "synthesis": Score() # Synthesis score for entities axis
+ },
+ "taxonomic_relations": {
+ "subclass": Score(),
+ "instanciation": Score(),
+ "synthesis": Score() # Synthesis score for taxonomic relations axis
+ },
+ "non_taxonomic_relations": {
+ "object_properties": Score(),
+ "data_properties": Score(),
+ "domains": Score(),
+ "ranges": Score(),
+ "synthesis": Score() # Synthesis score for non-taxonomic relations axis
+ },
+ "axioms": {
+ "restriction_axioms": Score(),
+ "synthesis": Score() # Synthesis score for axioms axis
+ }
}
+
#--------------------------------------------------------------------------
# Computing Method(s)
#--------------------------------------------------------------------------
- def calculate(self, reference_ontology, generated_ontology):
- """
- Compute scores (precision, recall, f1) for each ontology element category.
-
- Args:
- - reference_ontology: Ontology object representing the reference ontology.
- - generated_ontology: Ontology object representing the generated ontology.
- """
- methods = [
- ("class", "get_classes"),
- ("object_property", "get_object_properties"),
- # Additional methods can be uncommented as needed
- #("data_property", "get_data_properties"),
- #("restriction", "get_restrictions"),
- ("individual", "get_individuals"),
- #("annotation", "get_annotations")
- ]
+ @staticmethod
+ def deduplicate_elements(elements, comparison_function):
+ unique_elements = []
+ for elem in elements:
+ if not any([comparison_function(elem, unique_elem) for unique_elem in unique_elements]):
+ unique_elements.append(elem)
+ return unique_elements
+
+ def compute_entity_scores(self, reference_ontology, generated_ontology):
+ entity_methods = {
+ "classes": ("get_classes", Ontology.compare_entity_names),
+ "object_properties": ("get_object_properties", Ontology.compare_entity_names),
+ "individuals": ("get_individuals", Ontology.compare_entity_names)
+ }
y_true_overall = []
y_pred_overall = []
- for score_name, method_name in methods:
- reference_elements = set([elem.name() for elem in getattr(reference_ontology, method_name)()])
- generated_elements = set([elem.name() for elem in getattr(generated_ontology, method_name)()])
+ for score_name, (method_name, comparison_function) in entity_methods.items():
+ reference_elements_raw = getattr(reference_ontology, method_name)()
+ generated_elements_raw = getattr(generated_ontology, method_name)()
+
+ reference_elements = Metrics.deduplicate_elements(reference_elements_raw, comparison_function)
+ generated_elements = Metrics.deduplicate_elements(generated_elements_raw, comparison_function)
- all_elements = reference_elements.union(generated_elements)
- y_true = [1 if elem in reference_elements else 0 for elem in all_elements]
- y_pred = [1 if elem in generated_elements else 0 for elem in all_elements]
+ all_elements = list(set(reference_elements + generated_elements))
+ all_elements = Metrics.deduplicate_elements(all_elements, comparison_function)
+ y_true = [1 if any([comparison_function(elem, ref_elem) for ref_elem in reference_elements]) else 0 for elem in all_elements]
+ y_pred = [1 if any([comparison_function(elem, gen_elem) for gen_elem in generated_elements]) else 0 for elem in all_elements]
- self.scores[score_name].compute(y_true, y_pred)
+ self.scores["entities"][score_name].compute(y_true, y_pred)
y_true_overall.extend(y_true)
y_pred_overall.extend(y_pred)
- self.scores["overall"].compute(y_true_overall, y_pred_overall)
+ self.scores["entities"]["synthesis"].compute(y_true_overall, y_pred_overall)
@@ -90,7 +105,8 @@ class Metrics:
"""
Prints the scores (precision, recall, f1) for each ontology element category.
"""
- for element, score in self.scores.items():
- print(f"Metrics for {element.capitalize()}:")
+ entity_scores = self.scores["entities"]
+ for element, score in entity_scores.items():
+ print(f"Metrics for {element.capitalize()} (Entity axis):")
print(score)
print("----------------------------")
diff --git a/ontoScorer/ontology.py b/ontoScorer/ontology.py
index 942993d..64fff47 100644
--- a/ontoScorer/ontology.py
+++ b/ontoScorer/ontology.py
@@ -122,6 +122,11 @@ class Ontology:
# Comparison Method(s)
#--------------------------------------------------------------------------
+ @staticmethod
+ def compare_entity_names(entity1, entity2):
+ return entity1.name() == entity2.name()
+
+
def compare_to(self, other_ontology) -> tuple:
"""Compare classes of the current ontology with another."""
self_classes = {c.name() for c in self.classes}
diff --git a/ontoScorer/report.py b/ontoScorer/report.py
index a5df58a..9cc27ef 100644
--- a/ontoScorer/report.py
+++ b/ontoScorer/report.py
@@ -1,42 +1,87 @@
#!/usr/bin/python3.10
# -*-coding:Utf-8 -*
-#==============================================================================
-# ontoScorer: [brief description of the module]
-#------------------------------------------------------------------------------
-# Detailed module description, if needed
-#==============================================================================
+"""
+Ontology Report Module
+------------------------------------------------------------------------------
+This module provides a means to generate detailed reports that compare a
+reference ontology with a generated ontology. It uses various metrics to
+highlight the similarities and differences between the two ontologies,
+facilitating a comprehensive evaluation of ontology generation techniques.
+
+The reports can be used for debugging, optimization, and academic purposes,
+helping developers and researchers to better understand the quality and
+characteristics of generated ontologies in comparison to a reference.
+
+Classes:
+ - Report: Main class that represents an ontology comparison report.
+"""
class Report:
- def __init__(self, reference_ontology, generated_ontology, comparison_result, metrics):
+ """
+ A class used to generate detailed reports on the comparison of a
+ reference ontology with a generated ontology.
+
+ Attributes:
+ ------------
+ reference_ontology : obj
+ The reference ontology against which the generated ontology is compared.
+ generated_ontology : obj
+ The ontology that has been generated and needs to be evaluated.
+ metrics : obj
+ The metrics used to evaluate the generated ontology against the reference.
+
+ Methods:
+ ---------
+ generate() -> str:
+ Produces a string representation of the report, detailing the comparison
+ results and associated metrics.
+ """
+
+ def __init__(self, reference_ontology, generated_ontology, metrics):
+ """
+ Initializes the Report with a reference ontology, generated ontology,
+ and comparison metrics.
+ """
self.reference_ontology = reference_ontology
self.generated_ontology = generated_ontology
- self.comparison_result = comparison_result
self.metrics = metrics
- def generate(self):
- report_str = "=== Ontology Evaluation Report ===\n"
- report_str += f"\nReference Ontology: {self.reference_ontology.path}"
- report_str += f"\nNumber of classes in Reference Ontology: {len(self.reference_ontology.classes)}"
+ def _generate_evaluation_metrics(self, category_name):
+ scores_str = f"\n\n== {category_name.replace('_', ' ').capitalize()} ==\n"
+
+ category_scores = self.metrics.scores[category_name]
+ for element, score in category_scores.items():
+ if element != "synthesis": # We'll print synthesis separately
+ scores_str += f"\nMetrics for {element.replace('_', ' ').capitalize()}:\n"
+ scores_str += f'Precision: {score._format_metric(score.precision)}\n'
+ scores_str += f'Recall: {score._format_metric(score.recall)}\n'
+ scores_str += f'F1 Score: {score._format_metric(score.f1)}\n'
+ scores_str += f'Total Elements: {score.total_elements}\n'
+ scores_str += f'Matched Elements: {score.matched_elements}\n'
+
+ # Print synthesis score at the end for the category
+ synthesis_score = category_scores["synthesis"]
+ scores_str += "\nOverall Metrics (Synthesis):\n"
+ scores_str += f'Precision: {synthesis_score._format_metric(synthesis_score.precision)}\n'
+ scores_str += f'Recall: {synthesis_score._format_metric(synthesis_score.recall)}\n'
+ scores_str += f'F1 Score: {synthesis_score._format_metric(synthesis_score.f1)}\n'
+
+ return scores_str
- report_str += f"\n\nGenerated Ontology: {self.generated_ontology.path}"
- report_str += f"\nNumber of classes in Generated Ontology: {len(self.generated_ontology.classes)}"
- # Comparison of the number of classes
- report_str += "\n\nComparison Result: "
- if len(self.reference_ontology.classes) > len(self.generated_ontology.classes):
- report_str += "The generated ontology has fewer classes than the reference ontology."
- elif len(self.reference_ontology.classes) < len(self.generated_ontology.classes):
- report_str += "The generated ontology has more classes than the reference ontology."
- else:
- report_str += "The generated ontology and the reference ontology have the same number of classes."
+ def generate(self):
+ report_str = "=== Ontology Evaluation Report ===\n"
- report_str += "\n\nEvaluation Metrics:"
- report_str += f'\nPrecision: {self.metrics.scores["overall"]["precision"]}'
- report_str += f'\nRecall: {self.metrics.scores["overall"]["recall"]}'
- report_str += f'\nF1 Score: {self.metrics.scores["overall"]["f1"]}'
+ # Introduction
+ report_str += "\nComparing Reference Ontology with Generated Ontology.\n"
+
+ # Detailed Evaluation Metrics
+ for category in self.metrics.scores.keys():
+ report_str += self._generate_evaluation_metrics(category)
return report_str
+
diff --git a/ontoScorer/scorer.py b/ontoScorer/scorer.py
index 763d930..f1e6b8d 100644
--- a/ontoScorer/scorer.py
+++ b/ontoScorer/scorer.py
@@ -17,14 +17,13 @@ class OntoScorer:
self.generated_ontology = Ontology(generated_ontology_path)
self.metrics = Metrics()
- def compare(self):
+ def compute_metrics(self):
self.comparison_result = self.reference_ontology.compare_to(self.generated_ontology)
- self.metrics.calculate(self.reference_ontology, self.generated_ontology)
+ self.metrics.compute_entity_scores(self.reference_ontology, self.generated_ontology)
def generate_report(self):
report = Report(self.reference_ontology,
self.generated_ontology,
- self.comparison_result,
self.metrics)
print(report.generate())
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index 0eb3365..de92d2f 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -27,8 +27,8 @@ class TestMetrics(unittest.TestCase):
self.metrics = Metrics()
- def test_calculate_scores(self):
- self.metrics.calculate(self.onto1, self.onto2)
+ def test_computes_scores(self):
+ self.metrics.compute_entity_scores(self.onto1, self.onto2)
for element, score in self.metrics.scores.items():
if score.total_elements == 0:
self.assertIsNone(score.precision, f"Precision for {element} should be None when no elements are present")
@@ -41,7 +41,7 @@ class TestMetrics(unittest.TestCase):
def test_print_scores(self):
- self.metrics.calculate(self.onto1, self.onto2)
+ self.metrics.compute_entity_scores(self.onto1, self.onto2)
print()
self.metrics.print_scores()
--
GitLab