From 4cfa39bde0bf74f89db6f456ef0ddd9e0a8b5872 Mon Sep 17 00:00:00 2001
From: daxid <david.rouquet@tetras-libre.fr>
Date: Thu, 25 Nov 2021 09:51:35 +0000
Subject: [PATCH] Version avec verif pour les phrases venant d'UNseL Inter
---
unsel_dashboard-iterate.ipynb | 243 ++++++++++------------------------
1 file changed, 67 insertions(+), 176 deletions(-)
diff --git a/unsel_dashboard-iterate.ipynb b/unsel_dashboard-iterate.ipynb
index e0dae34..c55b607 100644
--- a/unsel_dashboard-iterate.ipynb
+++ b/unsel_dashboard-iterate.ipynb
@@ -442,8 +442,6 @@
" INSERT DATA {{<exec_instance> rdf:type {}}}\n",
"\"\"\"\n",
"\n",
- "\n",
- "\n",
"def owl2vowl(ttlFilePath, importList=[]):\n",
" # Run java parser\n",
" if importList == []:\n",
@@ -472,98 +470,7 @@
" inferResult = shaclInfer(work_file, 'infer') # apply SHACL inference\n",
" graph.parse(data=inferResult) # update graph with inference\n",
" graph.serialize(destination=dest_file, base=base_ref, format='turtle') # serialize graph after inference\n",
- " return graph, inferResult\n",
- " \n",
- " \n",
- "#owl2vowl('/opt/dashboards/printtatic/extraction.ttl', importList=['system-ontology.ttl'])\n",
- "def createOnto(uuidStr):\n",
- " \n",
- " # -- Initialization (creation of extraction graph)\n",
- " step = 'init_graph'\n",
- " # -- old --- step_ref = \"cts:\" + step\n",
- " dest_file = workDir + uuidStr + '-' + step + \".ttl\"\n",
- " base_ref = \"http://\" + uuidStr + '/' + step\n",
- " graph = createTenetGraph(uuidStr)\n",
- " graph.serialize(destination=dest_file, base=base_ref, format='turtle')\n",
- " \n",
- " # -- Extraction\n",
- " graph, _ = applyInferStep(uuidStr, graph, 'preprocessing')\n",
- " graph, _ = applyInferStep(uuidStr, graph, 'net_extension')\n",
- " graph, finalInferResult = applyInferStep(uuidStr, graph, 'generation_dga_patch')\n",
- " # -- old --- graph, finalInferResult = applyInferStep(uuidStr, graph, 'generation')\n",
- " \n",
- " # -- dev --- Generation step by step\n",
- " # -- old --- graph, _ = applyInferStep(uuidStr, graph, 'class_generation')\n",
- " # -- old --- graph, _ = applyInferStep(uuidStr, graph, 'relation_generation_1')\n",
- " # -- old --- graph, _ = applyInferStep(uuidStr, graph, 'relation_generation_2')\n",
- " # -- old --- graph, _ = applyInferStep(uuidStr, graph, 'relation_generation_3_1')\n",
- " # -- old --- graph, finalInferResult = applyInferStep(uuidStr, graph, 'relation_generation_3_2')\n",
- " # -- old --- graph, finalInferResult = applyInferStep(uuidStr, graph, 'relation_generation')\n",
- " # -- dev ---\n",
- " \n",
- " # -- Preprocessing\n",
- " # -- old --- step = 'preprocessing'\n",
- " # -- old --- step_ref = \"cts:\" + step\n",
- " # -- old --- dest_file = step + \".ttl\"\n",
- " # -- old --- base_ref = \"http://\" + step\n",
- " # -- old --- graph.update(clearExecutionInstances)\n",
- " # -- old --- graph.update(addExecutionInstance.format(step_ref)) # ex. : step = 'cts:generation'\n",
- " # -- old --- graph.serialize(destination=dest_file, base=base_ref, format='turtle')\n",
- " # -- old --- work_file = dest_file\n",
- " # -- old --- inferResult1 = shaclInfer(work_file, 'infer')\n",
- " # -- old --- graph.parse(data=inferResult1)\n",
- " # -- old --- graph.serialize(destination=dest_file, base=base_ref, format='turtle')\n",
- " \n",
- " # -- Net Extension \n",
- " # -- old --- step = 'net_extension'\n",
- " # -- old --- step_ref = \"cts:\" + step\n",
- " # -- old --- dest_file = step + \".ttl\"\n",
- " # -- old --- base_ref = \"http://\" + step\n",
- " # -- old --- graph.update(clearExecutionInstances)\n",
- " # -- old --- graph.update(addExecutionInstance.format(step_ref)) # ex. : step = 'cts:generation'\n",
- " # -- old --- graph.serialize(destination=dest_file, base=base_ref, format='turtle')\n",
- " # -- old --- work_file = dest_file\n",
- " # -- old --- inferResult2 = shaclInfer(work_file, 'infer')\n",
- " # -- old --- graph.parse(data=inferResult2)\n",
- " # -- old --- graph.serialize(destination=dest_file, base=base_ref, format='turtle')\n",
- " \n",
- " # -- Generation\n",
- " # -- old --- step = 'generation'\n",
- " # -- old --- step_ref = \"cts:\" + step\n",
- " # -- old --- dest_file = step + \".ttl\"\n",
- " # -- old --- base_ref = \"http://\" + step\n",
- " # -- old --- graph.update(clearExecutionInstances)\n",
- " # -- old --- graph.update(addExecutionInstance.format(step_ref)) # ex. : step = 'cts:generation'\n",
- " # -- old --- graph.serialize(destination=dest_file, base=base_ref, format='turtle') # serialize graph before inference\n",
- " # -- old --- work_file = dest_file\n",
- " # -- old --- finalInferResult = shaclInfer(work_file, 'infer') # apply SHACL inference\n",
- " # -- old --- graph.parse(data=finalInferResult) # update graph with inference\n",
- " # -- old --- graph.serialize(destination=dest_file, base=base_ref, format='turtle') # serialize graph after inference\n",
- " \n",
- " # -- Result\n",
- " # -- old --- file_ref = CONFIG_DIR + schema_file\n",
- " # -- old --- sys_frame_onto = FRAME_DIR + frame_file\n",
- " factoidPath = storeBase+uuidStr+'/current/'+uuidStr+'_factoid.ttl'\n",
- " with open(factoidPath, 'w') as outfile:\n",
- " outfile.write(finalInferResult)\n",
- " outfile.close()\n",
- " \n",
- " owl2vowl(factoidPath)#, importList=[sys_frame_onto])\n",
- " \n",
- " #result_graph.parse(data=finalInferResult) # update with inferences resulting of extraction final step\n",
- " #factoidPath = storeBase+uuidStr+'/current/'+uuidStr+'_factoid.ttl'\n",
- " #result_graph.serialize(destination=factoidPath, base=base_ref, format='turtle') \n",
- " \n",
- " # On exécute ensuite seulement les règles de génération pour récupérer seulement le nécesaire à la visualisation\n",
- " # -- old --- graph.update(clearExecutionInstances)\n",
- " # -- old --- graph.update(addExecutionInstance.format('cts:generation'))\n",
- " # -- old --- graph.serialize(destination=\"tmp.ttl\", base=\"http://tmp\", format='turtle')\n",
- " \n",
- " # -- old --- factoidPath = storeBase+uuidStr+'/current/'+uuidStr+'_factoid.ttl'\n",
- " # -- old --- with open(factoidPath, 'w') as outfile:\n",
- " # -- old --- inferResult3 = shaclInfer('tmp.ttl', 'infer')\n",
- " # -- old --- outfile.write(inferResult3)\n",
- " # -- old --- outfile.close()\n"
+ " return graph, inferResult"
]
},
{
@@ -641,7 +548,8 @@
"\n",
"SELECT (CONCAT(?messageTMP, ?sentenceList, ')') AS ?message) ?sentenceList\n",
"WHERE {\n",
- "SELECT ?messageTMP (GROUP_CONCAT(?sentenceId; SEPARATOR=\", \") AS ?sentenceList)\n",
+ "FILTER(regex(?sentenceList, \"##ID##\" ))\n",
+ "{SELECT ?messageTMP (GROUP_CONCAT(?sentenceId; SEPARATOR=\", \") AS ?sentenceList)\n",
"WHERE {\t\n",
"\tSELECT DISTINCT ?messageTMP ?sentenceId\n",
"\tWHERE {\n",
@@ -649,7 +557,7 @@
"\t\t?subEntity rdfs:subClassOf ?this ; rdfs:label ?subEntityLabel ; sys:from_structure ?subEntitySentence .\n",
"\t\t?this rdfs:label ?thisLabel ; sys:from_structure ?thisSentence .\n",
" \tBIND(CONCAT('\"', ?subEntityLabel, '\" est la seule sous classe de \"', ?thisLabel, '\". Ces classes sont-elles équivalentes ? <br/>(voir les exigences ') AS ?messageTMP)\n",
- " \t\t{BIND(\"##ID##\" AS ?thisSentenceId)\n",
+ " \t\t{\n",
" ?thisSentence unl:has_id ?thisSentenceId .\n",
"\t\t\tBIND (?thisSentenceId AS ?sentenceId)} \n",
"\t\tUNION \n",
@@ -660,9 +568,9 @@
"\t\tWHERE {\n",
" \t \t?this rdfs:subClassOf+ sys:Structure .\n",
"\t\t\t\t?subClass rdfs:subClassOf ?this\n",
- "\t\t} GROUP BY ?this } \n",
+ "\t\t} GROUP BY ?this }\n",
"\t} ORDER BY ?sentenceId\n",
- "} GROUP BY ?messageTMP\n",
+ "} GROUP BY ?messageTMP }\n",
"}'''\n",
"\n",
"unfeaturedDomainOrRangeQuery = '''PREFIX owl: <http://www.w3.org/2002/07/owl#>\n",
@@ -784,41 +692,53 @@
"g = loadFactoids(storeBase)\n",
"sparqlUpdate.setQuery('INSERT DATA {'+g.serialize(format='nt')+'}')\n",
"sparqlLog = sparqlUpdate.query()\n",
+ "g.serialize(destination='/opt/dashboards/store/extraction2.ttl', format='turtle')\n",
+ "owl2vowl('/opt/dashboards/store/extraction2.ttl')\n",
+ "#factoidList = [x.replace('..','/opt/dashboards') for x in glob(storeBase+\"/*/current/*factoid.ttl\")]\n",
+ "#owl2vowl(FRAME_DIR + frame_file, importList=factoidList)\n",
"\n",
- "#g.serialize(destination='/opt/dashboards/store/SRSA-IP_demo.ttl', format='turtle')\n",
- "\n",
- "\n",
- "#errorReqDic = {}\n",
- "#warningReqDic = {}\n",
- "#infoReqDic = {}\n",
- "\n",
- "\n",
- "#possibleUnderspecification = g.query(possibleUnderspecificationQuery)\n",
- "#possibleClassEquivalence = g.query(possibleClassEquivalenceQuery)\n",
- "#unfeaturedDomainOrRange = g.query(unfeaturedDomainOrRangeQuery)\n",
- "\n",
+ "def createVerificationMessages(directory):\n",
+ " reqId = directory.split('/')[-1]\n",
+ " for messageType in ['error', 'warning', 'info']:\n",
+ " #print(messageType)\n",
+ " messagesStr = ''\n",
+ " for queryTMP in queryTypeDic[messageType] :\n",
+ " query = queryTMP.replace(\"##ID##\",reqId)\n",
+ " sparqlQuery.setQuery(query)\n",
+ " results = sparqlQuery.query().convert()\n",
+ " for result in results[\"results\"][\"bindings\"]:\n",
+ " message = result[\"message\"][\"value\"]\n",
+ " if message not in messagesStr :\n",
+ " #print(\"printing\")\n",
+ " messagesStr+=message+\"\\n\"\n",
+ " outFile = open(directory+'/current/'+reqId+'_'+messageType+'Messages.txt', 'w')\n",
+ " outFile.write(messagesStr)\n",
+ " outFile.close() \n",
"\n",
- "\n",
- "#for r in possibleUnderspecification : \n",
- "# if str(r['thisId']) in warningReqDic:\n",
- "# warningReqDic[str(r['thisId'])] += [str(r['message'])]\n",
- "# else:\n",
- "# warningReqDic[str(r['thisId'])] = [str(r['message'])] \n",
- "\n",
- "#for r in possibleClassEquivalence : \n",
- "# sentenceList = str(r['sentenceList']).split(', ')\n",
- "# for sentence in sentenceList:\n",
- "# if sentence in errorReqDic:\n",
- "# errorReqDic[sentence] += [str(r['message'])]\n",
- "# else:\n",
- "# errorReqDic[sentence] = [str(r['message'])] \n",
- "\n",
- "#for r in unfeaturedDomainOrRange : \n",
- "# if str(r['sentenceId']) in infoReqDic:\n",
- "# infoReqDic[str(r['sentenceId'])] += [str(r['message'])]\n",
- "# else:\n",
- "# infoReqDic[str(r['sentenceId'])] = [str(r['message'])] \n",
- "\n"
+ "def createOnto(uuidStr): \n",
+ " # -- Initialization (creation of extraction graph)\n",
+ " step = 'init_graph'\n",
+ " dest_file = workDir + uuidStr + '-' + step + \".ttl\"\n",
+ " base_ref = \"http://\" + uuidStr + '/' + step\n",
+ " graph = createTenetGraph(uuidStr)\n",
+ " graph.serialize(destination=dest_file, base=base_ref, format='turtle') \n",
+ " # -- Extraction\n",
+ " graph, _ = applyInferStep(uuidStr, graph, 'preprocessing')\n",
+ " graph, _ = applyInferStep(uuidStr, graph, 'net_extension')\n",
+ " graph, finalInferResult = applyInferStep(uuidStr, graph, 'generation_dga_patch')\n",
+ " # -- Write result \n",
+ " factoidPath = storeBase+uuidStr+'/current/'+uuidStr+'_factoid.ttl'\n",
+ " with open(factoidPath, 'w') as outfile:\n",
+ " outfile.write(finalInferResult)\n",
+ " outfile.close()\n",
+ " # -- Webvowl\n",
+ " owl2vowl(factoidPath)\n",
+ " # -- Verification\n",
+ " factoidGraph = Graph()\n",
+ " factoidGraph.parse(factoidPath)\n",
+ " sparqlUpdate.setQuery('INSERT DATA {'+factoidGraph.serialize(format='nt')+'}')\n",
+ " sparqlLog = sparqlUpdate.query()\n",
+ " createVerificationMessages(storeBase+uuidStr)"
]
},
{
@@ -848,22 +768,7 @@
" for directory in dirList:\n",
" if directory.split('/')[-1] != '0_NONE':\n",
" print(directory)\n",
- " reqId = directory.split('/')[-1]\n",
- " for messageType in ['error', 'warning', 'info']:\n",
- " #print(messageType)\n",
- " messagesStr = ''\n",
- " for queryTMP in queryTypeDic[messageType] :\n",
- " query = queryTMP.replace(\"##ID##\",reqId)\n",
- " sparqlQuery.setQuery(query)\n",
- " results = sparqlQuery.query().convert()\n",
- " for result in results[\"results\"][\"bindings\"]:\n",
- " message = result[\"message\"][\"value\"]\n",
- " if message not in messagesStr :\n",
- " #print(\"printing\")\n",
- " messagesStr+=message+\"\\n\"\n",
- " outFile = open(directory+'/current/'+reqId+'_'+messageType+'Messages.txt', 'w')\n",
- " outFile.write(messagesStr)\n",
- " outFile.close() \n",
+ " createVerificationMessages(directory) \n",
"\n",
"#updateAllVerificationMessages()\n",
"\n",
@@ -1039,19 +944,6 @@
"metadata": {},
"outputs": [],
"source": [
- "#######################################################################################################\n",
- "# Analyse avec UNseL-inter\n",
- "#######################################################################################################\n",
- "def extractOnClick(event):\n",
- " uuidStr = \"COMP\"+str(uuid.uuid4())\n",
- " createFolderFromUnselInter(uuidStr)\n",
- " selectDir.options[uuidStr] = uuidStr\n",
- " selectDir.value = uuidStr\n",
- " createOnto(uuidStr)\n",
- "\n",
- "buttonExtract = pn.widgets.Button(name=\"Construire l'ontologie de la phrase (factoïd)\", width=300)\n",
- "buttonExtract.param.watch(extractOnClick, 'clicks')\n",
- "\n",
"#######################################################################################################\n",
"# Navigation Interface\n",
"#######################################################################################################\n",
@@ -1075,6 +967,21 @@
"selectDir.value = '0_NONE'\n",
"dir_selector = dict(directory=selectDir)#, save=saveButton)\n",
"\n",
+ "#######################################################################################################\n",
+ "# Analyse avec UNseL-inter\n",
+ "#######################################################################################################\n",
+ "def extractOnClick(event):\n",
+ " uuidStr = \"COMP\"+str(uuid.uuid4())\n",
+ " createFolderFromUnselInter(uuidStr)\n",
+ " createOnto(uuidStr) \n",
+ " selectDir.options[uuidStr] = uuidStr\n",
+ " selectDir.value = uuidStr\n",
+ "##################################################\n",
+ " \n",
+ "buttonExtract = pn.widgets.Button(name=\"Construire l'ontologie de la phrase (factoïd)\", width=300)\n",
+ "buttonExtract.param.watch(extractOnClick, 'clicks')\n",
+ "\n",
+ "\n",
"unlNavigatorPane = pn.interact(lambda directory : main_pane(directory), **dir_selector)\n",
"\n",
"#unl_graph_pane = pn.interact(lambda button : compute_unl_graph_pane(button), **saveButtonDic)\n",
@@ -1136,22 +1043,6 @@
"# uuidStr = d.replace('/opt/dashboards/store/CCTP-SRSA-IP-20210831/','')\n",
"# createOnto(uuidStr)"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "47f67a4b-4d6c-4192-93b1-e697fd2c0e73",
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "18d75469-934f-4021-9126-ca0c1dcd4d98",
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
--
GitLab