Add scorer, .gitignore

.gitignore

+__pycache__/

scorer.py

+# Compares some results (e.g. output of Tenet) w.r.t a corpus
+# with the expected ODRL representations of the corpus
+# 
+# ODRL files in the corpus directory should have the same name as in the
+# output directory
+
+import os
+import sys
+from rdflib import Graph
+
+class ODRL:
+	
+	def __init__(self):
+		self.has_perm = False
+		self.has_obl = False
+		self.has_proh = False
+		self.perm = []
+		self.obl = []
+		self.proh = []
+	
+	""" Parses the content of an ODRL file and adds its contents to the ODRL """
+	def parse(self, odrl_fname):
+		graph = Graph()
+		graph.parse(odrl_fname)
+		odrl_ns = "http://www.w3.org/ns/odrl/2/"
+		ccrel_ns = "http://creativecommons.org/ns#"
+		perm_node = None
+		obl_node = None
+		proh_node = None
+		actions = {} # action listing for every blank node
+		
+		# List actions and modalities
+		for (src, prop, tgt) in graph:
+			src, prop, tgt = str(src), str(prop), str(tgt)
+			if prop == odrl_ns + "permission":
+				self.has_perm = True
+				perm_node = tgt
+			elif prop == odrl_ns + "obligation":
+				self.has_obl = True
+				obl_node = tgt
+			elif prop == odrl_ns + "prohibition":
+				self.has_proh = True
+				proh_node = tgt
+			elif prop == odrl_ns + "action":
+				if src not in actions: actions[src] = []
+				actions[src].append(tgt)
+		
+		# Link modalities with actions
+		for node in actions:
+			if node == perm_node: mod = self.perm
+			elif node == obl_node: mod = self.obl
+			elif node == proh_node: mod = self.proh
+			else: print("Warning: ill-formed ODRL")
+			for act in actions[node]:
+				mod.append(act)
+	
+	def actions(self):
+		acts = []
+		for act in self.perm: acts.append(act)
+		for act in self.obl: acts.append(act)
+		for act in self.proh: acts.append(act)
+		return acts
+	
+	def __str__(self):
+		s = ""
+		if self.has_perm:
+			s += "Permissions:\n"
+			for act in self.perm:
+				s += f"\t{act}\n"
+		if self.has_obl:
+			s += "Obligations:\n"
+			for act in self.obl:
+				s += f"\t{act}\n"
+		if self.has_proh:
+			s += "Prohibitions:\n"
+			for act in self.proh:
+				s += f"\t{act}\n"
+		return s
+
+
+class Scores:
+
+	def __init__(self):
+		self.scores = {}
+		
+	def check_and_add_criterion(self, crit):
+		if crit not in self.scores:
+			self.scores[crit] = {"tp": 0, "fp": 0, "fn": 0}
+	
+	""" Adds a single measure on the score.
+	    tgt (bool): real value
+	    output (bool): measured value """
+	def add_measure(self, crit, tgt, output):
+		self.check_and_add_criterion(crit)
+		if tgt and output:
+			self.scores[crit]["tp"] += 1
+		elif not tgt and output:
+			self.scores[crit]["fp"] += 1
+		elif tgt and not output:
+			self.scores[crit]["fn"] += 1
+	
+	def get_precision(self, crit):
+		return self.scores[crit]["tp"]/(self.scores[crit]["tp"] + self.scores[crit]["fp"])
+	
+	def get_recall(self, crit):
+		return self.scores[crit]["tp"]/(self.scores[crit]["tp"] + self.scores[crit]["fn"])
+	
+	def get_total_precision(self):
+		tot_tp = 0
+		tot_fp = 0
+		for crit in self.scores:
+			tot_tp += self.scores[crit]["tp"]
+			tot_fp += self.scores[crit]["fp"]
+		return tot_tp/(tot_tp + tot_fp)
+			
+	def get_total_recall(self):
+		tot_tp = 0
+		tot_fn = 0
+		for crit in self.scores:
+			tot_tp += self.scores[crit]["tp"]
+			tot_fn += self.scores[crit]["fn"]
+		return tot_tp/(tot_tp + tot_fn)
+	
+	def __str__(self):
+		s = ""
+		for crit in self.scores:
+			s += f"{crit}:\n"
+			s += f'\tTP = {self.scores[crit]["tp"]}, FP = {self.scores[crit]["fp"]}, FN = {self.scores[crit]["fn"]}\n'
+			s += f'\tPrecision = {self.get_precision(crit)}, Recall = {self.get_recall(crit)}\n'
+		s += f'Total precision = {self.get_total_precision()}\n'
+		s += f'Total recall = {self.get_total_recall()}\n'
+		return s
+		
+
+
+def add_comp_modalities(odrl_target, odrl_output, scores):
+	scores.add_measure("permission", odrl_target.has_perm, odrl_output.has_perm)
+	scores.add_measure("obligation", odrl_target.has_obl, odrl_output.has_obl)
+	scores.add_measure("prohibition", odrl_target.has_proh, odrl_output.has_proh)
+
+def add_comp_actions(odrl_target, odrl_output, scores):
+	actions_target = odrl_target.actions()
+	actions_output = odrl_output.actions()
+	
+	for action in actions_output:
+		scores.add_measure(action, action in actions_target, True)
+	for action in actions_target:
+		if action not in actions_output: # Don't reconsider True, True
+			scores.add_measure(action, True, False)
+
+def add_comp_global(odrl_target, odrl_output, scores):
+	same = set(odrl_target.perm) == set(odrl_output.perm)
+	same = same and set(odrl_target.obl) == set(odrl_output.obl)
+	same = same and set(odrl_target.proh) == set(odrl_output.proh)
+	scores.add_measure("global", True, same)
+
+
+""" path_target: path to the ODRL files of the corpus
+    path_output: path to the ODRL files of the output """
+def main_scorer(path_target, path_output):
+	
+	scores_modalities = Scores()
+	scores_actions = Scores()
+	scores_global = Scores()
+	
+	for fname in os.listdir(path_odrl):
+		if fname.endswith(".ttl"):
+		
+			odrl_target = ODRL()
+			odrl_target.parse(path_odrl + fname)
+			odrl_output = ODRL()
+			odrl_output.parse(path_output + fname)
+			
+			add_comp_modalities(odrl_target, odrl_output, scores_modalities)
+			add_comp_actions(odrl_target, odrl_output, scores_actions)
+			add_comp_global(odrl_target, odrl_output, scores_global)
+	
+	return scores_modalities, scores_actions, scores_global
+
+
+
+if __name__ == "__main__":
+
+	if len(sys.argv) < 3:
+		print(f"Usage: python3 {sys.argv[0]} <path_corpus_odrl> <path_output_odrl")
+		exit(1)
+
+	path_target = sys.argv[1]
+	path_output = sys.argv[2]
+	
+	scores_modalities, scores_actions, scores_global = main_scorer(path_target, path_output)
+	
+	print(f"Modality scores:\n{scores_modalities}")
+	print(f"Actions scores:\n{scores_actions}")
+	print(f"Global scores:\n{scores_global}")