From e8ad18769e3ed91c7ed85d9b9f40c92f2d855fdd Mon Sep 17 00:00:00 2001 From: daxid <david.rouquet@tetras-libre.fr> Date: Fri, 19 Nov 2021 15:49:11 +0000 Subject: [PATCH] Working version req by req Still missing : verification for new req coming from UNseL-Inter. Debug Info query --- unsel_dashboard-iterate.ipynb | 1604 +++++++++++++++++++++++++++++++++ 1 file changed, 1604 insertions(+) create mode 100644 unsel_dashboard-iterate.ipynb diff --git a/unsel_dashboard-iterate.ipynb b/unsel_dashboard-iterate.ipynb new file mode 100644 index 0000000..2a3114d --- /dev/null +++ b/unsel_dashboard-iterate.ipynb @@ -0,0 +1,1604 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "160ab8ab-091e-4b00-b63a-c3746c71c540", + "metadata": {}, + "outputs": [ + { + "data": { + "application/javascript": [ + "\n", + "(function(root) {\n", + " function now() {\n", + " return new Date();\n", + " }\n", + "\n", + " var force = true;\n", + "\n", + " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n", + " root._bokeh_onload_callbacks = [];\n", + " root._bokeh_is_loading = undefined;\n", + " }\n", + "\n", + " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n", + " root._bokeh_timeout = Date.now() + 5000;\n", + " root._bokeh_failed_load = false;\n", + " }\n", + "\n", + " function run_callbacks() {\n", + " try {\n", + " root._bokeh_onload_callbacks.forEach(function(callback) {\n", + " if (callback != null)\n", + " callback();\n", + " });\n", + " } finally {\n", + " delete root._bokeh_onload_callbacks\n", + " }\n", + " console.debug(\"Bokeh: all callbacks have finished\");\n", + " }\n", + "\n", + " function load_libs(css_urls, js_urls, js_modules, callback) {\n", + " if (css_urls == null) css_urls = [];\n", + " if (js_urls == null) js_urls = [];\n", + " if (js_modules == null) js_modules = [];\n", + "\n", + " root._bokeh_onload_callbacks.push(callback);\n", + " if (root._bokeh_is_loading > 0) {\n", + " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", + " return null;\n", + " }\n", + " if (js_urls.length === 0 && js_modules.length === 0) {\n", + " run_callbacks();\n", + " return null;\n", + " }\n", + " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", + " root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length;\n", + "\n", + " function on_load() {\n", + " root._bokeh_is_loading--;\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", + " run_callbacks()\n", + " }\n", + " }\n", + "\n", + " function on_error() {\n", + " console.error(\"failed to load \" + url);\n", + " }\n", + "\n", + " for (var i = 0; i < css_urls.length; i++) {\n", + " var url = css_urls[i];\n", + " const element = document.createElement(\"link\");\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.rel = \"stylesheet\";\n", + " element.type = \"text/css\";\n", + " element.href = url;\n", + " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " var skip = [];\n", + " if (window.requirejs) {\n", + " window.requirejs.config({'packages': {}, 'paths': {}, 'shim': {}});\n", + " \n", + " }\n", + " for (var i = 0; i < js_urls.length; i++) {\n", + " var url = js_urls[i];\n", + " if (skip.indexOf(url) >= 0) { on_load(); continue; }\n", + " var element = document.createElement('script');\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.async = false;\n", + " element.src = url;\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.head.appendChild(element);\n", + " }\n", + " for (var i = 0; i < js_modules.length; i++) {\n", + " var url = js_modules[i];\n", + " if (skip.indexOf(url) >= 0) { on_load(); continue; }\n", + " var element = document.createElement('script');\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.async = false;\n", + " element.src = url;\n", + " element.type = \"module\";\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.head.appendChild(element);\n", + " }\n", + " if (!js_urls.length && !js_modules.length) {\n", + " on_load()\n", + " }\n", + " };\n", + "\n", + " function inject_raw_css(css) {\n", + " const element = document.createElement(\"style\");\n", + " element.appendChild(document.createTextNode(css));\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.4.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.4.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-2.4.1.min.js\", \"https://unpkg.com/@holoviz/panel@0.12.4/dist/panel.min.js\"];\n", + " var js_modules = [];\n", + " var css_urls = [\"https://unpkg.com/@holoviz/panel@0.12.4/dist/css/widgets.css\", \"https://unpkg.com/@holoviz/panel@0.12.4/dist/css/card.css\", \"https://unpkg.com/@holoviz/panel@0.12.4/dist/css/dataframe.css\", \"https://unpkg.com/@holoviz/panel@0.12.4/dist/css/alerts.css\", \"https://unpkg.com/@holoviz/panel@0.12.4/dist/css/loading.css\", \"https://unpkg.com/@holoviz/panel@0.12.4/dist/css/markdown.css\", \"https://unpkg.com/@holoviz/panel@0.12.4/dist/css/json.css\"];\n", + " var inline_js = [\n", + " function(Bokeh) {\n", + " inject_raw_css(\"\\n .bk.pn-loading.arcs:before {\\n background-image: url(\\\"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiBzdHlsZT0ibWFyZ2luOiBhdXRvOyBiYWNrZ3JvdW5kOiBub25lOyBkaXNwbGF5OiBibG9jazsgc2hhcGUtcmVuZGVyaW5nOiBhdXRvOyIgdmlld0JveD0iMCAwIDEwMCAxMDAiIHByZXNlcnZlQXNwZWN0UmF0aW89InhNaWRZTWlkIj4gIDxjaXJjbGUgY3g9IjUwIiBjeT0iNTAiIHI9IjMyIiBzdHJva2Utd2lkdGg9IjgiIHN0cm9rZT0iI2MzYzNjMyIgc3Ryb2tlLWRhc2hhcnJheT0iNTAuMjY1NDgyNDU3NDM2NjkgNTAuMjY1NDgyNDU3NDM2NjkiIGZpbGw9Im5vbmUiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCI+ICAgIDxhbmltYXRlVHJhbnNmb3JtIGF0dHJpYnV0ZU5hbWU9InRyYW5zZm9ybSIgdHlwZT0icm90YXRlIiByZXBlYXRDb3VudD0iaW5kZWZpbml0ZSIgZHVyPSIxcyIga2V5VGltZXM9IjA7MSIgdmFsdWVzPSIwIDUwIDUwOzM2MCA1MCA1MCI+PC9hbmltYXRlVHJhbnNmb3JtPiAgPC9jaXJjbGU+PC9zdmc+\\\")\\n }\\n \");\n", + " },\n", + " function(Bokeh) {\n", + " Bokeh.set_log_level(\"info\");\n", + " },\n", + " function(Bokeh) {} // ensure no trailing comma for IE\n", + " ];\n", + "\n", + " function run_inline_js() {\n", + " if ((root.Bokeh !== undefined) || (force === true)) {\n", + " for (var i = 0; i < inline_js.length; i++) {\n", + " inline_js[i].call(root, root.Bokeh);\n", + " }} else if (Date.now() < root._bokeh_timeout) {\n", + " setTimeout(run_inline_js, 100);\n", + " } else if (!root._bokeh_failed_load) {\n", + " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", + " root._bokeh_failed_load = true;\n", + " }\n", + " }\n", + "\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", + " run_inline_js();\n", + " } else {\n", + " load_libs(css_urls, js_urls, js_modules, function() {\n", + " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", + " run_inline_js();\n", + " });\n", + " }\n", + "}(window));" + ], + "application/vnd.holoviews_load.v0+json": "\n(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, js_modules, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n if (js_modules == null) js_modules = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls.length === 0 && js_modules.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n var skip = [];\n if (window.requirejs) {\n window.requirejs.config({'packages': {}, 'paths': {}, 'shim': {}});\n \n }\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n if (skip.indexOf(url) >= 0) { on_load(); continue; }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (var i = 0; i < js_modules.length; i++) {\n var url = js_modules[i];\n if (skip.indexOf(url) >= 0) { on_load(); continue; }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n if (!js_urls.length && !js_modules.length) {\n on_load()\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.4.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.4.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-2.4.1.min.js\", \"https://unpkg.com/@holoviz/panel@0.12.4/dist/panel.min.js\"];\n var js_modules = [];\n var css_urls = [\"https://unpkg.com/@holoviz/panel@0.12.4/dist/css/widgets.css\", \"https://unpkg.com/@holoviz/panel@0.12.4/dist/css/card.css\", \"https://unpkg.com/@holoviz/panel@0.12.4/dist/css/dataframe.css\", \"https://unpkg.com/@holoviz/panel@0.12.4/dist/css/alerts.css\", \"https://unpkg.com/@holoviz/panel@0.12.4/dist/css/loading.css\", \"https://unpkg.com/@holoviz/panel@0.12.4/dist/css/markdown.css\", \"https://unpkg.com/@holoviz/panel@0.12.4/dist/css/json.css\"];\n var inline_js = [\n function(Bokeh) {\n inject_raw_css(\"\\n .bk.pn-loading.arcs:before {\\n background-image: url(\\\"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiBzdHlsZT0ibWFyZ2luOiBhdXRvOyBiYWNrZ3JvdW5kOiBub25lOyBkaXNwbGF5OiBibG9jazsgc2hhcGUtcmVuZGVyaW5nOiBhdXRvOyIgdmlld0JveD0iMCAwIDEwMCAxMDAiIHByZXNlcnZlQXNwZWN0UmF0aW89InhNaWRZTWlkIj4gIDxjaXJjbGUgY3g9IjUwIiBjeT0iNTAiIHI9IjMyIiBzdHJva2Utd2lkdGg9IjgiIHN0cm9rZT0iI2MzYzNjMyIgc3Ryb2tlLWRhc2hhcnJheT0iNTAuMjY1NDgyNDU3NDM2NjkgNTAuMjY1NDgyNDU3NDM2NjkiIGZpbGw9Im5vbmUiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCI+ICAgIDxhbmltYXRlVHJhbnNmb3JtIGF0dHJpYnV0ZU5hbWU9InRyYW5zZm9ybSIgdHlwZT0icm90YXRlIiByZXBlYXRDb3VudD0iaW5kZWZpbml0ZSIgZHVyPSIxcyIga2V5VGltZXM9IjA7MSIgdmFsdWVzPSIwIDUwIDUwOzM2MCA1MCA1MCI+PC9hbmltYXRlVHJhbnNmb3JtPiAgPC9jaXJjbGU+PC9zdmc+\\\")\\n }\\n \");\n },\n function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\n function(Bokeh) {} // ensure no trailing comma for IE\n ];\n\n function run_inline_js() {\n if ((root.Bokeh !== undefined) || (force === true)) {\n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, js_modules, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "\n", + "if ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n", + " window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n", + "}\n", + "\n", + "\n", + " function JupyterCommManager() {\n", + " }\n", + "\n", + " JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n", + " if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n", + " var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n", + " comm_manager.register_target(comm_id, function(comm) {\n", + " comm.on_msg(msg_handler);\n", + " });\n", + " } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n", + " window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n", + " comm.onMsg = msg_handler;\n", + " });\n", + " } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n", + " google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n", + " var messages = comm.messages[Symbol.asyncIterator]();\n", + " function processIteratorResult(result) {\n", + " var message = result.value;\n", + " console.log(message)\n", + " var content = {data: message.data, comm_id};\n", + " var buffers = []\n", + " for (var buffer of message.buffers || []) {\n", + " buffers.push(new DataView(buffer))\n", + " }\n", + " var metadata = message.metadata || {};\n", + " var msg = {content, buffers, metadata}\n", + " msg_handler(msg);\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " return messages.next().then(processIteratorResult);\n", + " })\n", + " }\n", + " }\n", + "\n", + " JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n", + " if (comm_id in window.PyViz.comms) {\n", + " return window.PyViz.comms[comm_id];\n", + " } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n", + " var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n", + " var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n", + " if (msg_handler) {\n", + " comm.on_msg(msg_handler);\n", + " }\n", + " } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n", + " var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n", + " comm.open();\n", + " if (msg_handler) {\n", + " comm.onMsg = msg_handler;\n", + " }\n", + " } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n", + " var comm_promise = google.colab.kernel.comms.open(comm_id)\n", + " comm_promise.then((comm) => {\n", + " window.PyViz.comms[comm_id] = comm;\n", + " if (msg_handler) {\n", + " var messages = comm.messages[Symbol.asyncIterator]();\n", + " function processIteratorResult(result) {\n", + " var message = result.value;\n", + " var content = {data: message.data};\n", + " var metadata = message.metadata || {comm_id};\n", + " var msg = {content, metadata}\n", + " msg_handler(msg);\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " }) \n", + " var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n", + " return comm_promise.then((comm) => {\n", + " comm.send(data, metadata, buffers, disposeOnDone);\n", + " });\n", + " };\n", + " var comm = {\n", + " send: sendClosure\n", + " };\n", + " }\n", + " window.PyViz.comms[comm_id] = comm;\n", + " return comm;\n", + " }\n", + " window.PyViz.comm_manager = new JupyterCommManager();\n", + " \n", + "\n", + "\n", + "var JS_MIME_TYPE = 'application/javascript';\n", + "var HTML_MIME_TYPE = 'text/html';\n", + "var EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\n", + "var CLASS_NAME = 'output';\n", + "\n", + "/**\n", + " * Render data to the DOM node\n", + " */\n", + "function render(props, node) {\n", + " var div = document.createElement(\"div\");\n", + " var script = document.createElement(\"script\");\n", + " node.appendChild(div);\n", + " node.appendChild(script);\n", + "}\n", + "\n", + "/**\n", + " * Handle when a new output is added\n", + " */\n", + "function handle_add_output(event, handle) {\n", + " var output_area = handle.output_area;\n", + " var output = handle.output;\n", + " if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n", + " return\n", + " }\n", + " var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", + " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", + " if (id !== undefined) {\n", + " var nchildren = toinsert.length;\n", + " var html_node = toinsert[nchildren-1].children[0];\n", + " html_node.innerHTML = output.data[HTML_MIME_TYPE];\n", + " var scripts = [];\n", + " var nodelist = html_node.querySelectorAll(\"script\");\n", + " for (var i in nodelist) {\n", + " if (nodelist.hasOwnProperty(i)) {\n", + " scripts.push(nodelist[i])\n", + " }\n", + " }\n", + "\n", + " scripts.forEach( function (oldScript) {\n", + " var newScript = document.createElement(\"script\");\n", + " var attrs = [];\n", + " var nodemap = oldScript.attributes;\n", + " for (var j in nodemap) {\n", + " if (nodemap.hasOwnProperty(j)) {\n", + " attrs.push(nodemap[j])\n", + " }\n", + " }\n", + " attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n", + " newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n", + " oldScript.parentNode.replaceChild(newScript, oldScript);\n", + " });\n", + " if (JS_MIME_TYPE in output.data) {\n", + " toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n", + " }\n", + " output_area._hv_plot_id = id;\n", + " if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n", + " window.PyViz.plot_index[id] = Bokeh.index[id];\n", + " } else {\n", + " window.PyViz.plot_index[id] = null;\n", + " }\n", + " } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", + " var bk_div = document.createElement(\"div\");\n", + " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", + " var script_attrs = bk_div.children[0].attributes;\n", + " for (var i = 0; i < script_attrs.length; i++) {\n", + " toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n", + " }\n", + " // store reference to server id on output_area\n", + " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", + " }\n", + "}\n", + "\n", + "/**\n", + " * Handle when an output is cleared or removed\n", + " */\n", + "function handle_clear_output(event, handle) {\n", + " var id = handle.cell.output_area._hv_plot_id;\n", + " var server_id = handle.cell.output_area._bokeh_server_id;\n", + " if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n", + " var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n", + " if (server_id !== null) {\n", + " comm.send({event_type: 'server_delete', 'id': server_id});\n", + " return;\n", + " } else if (comm !== null) {\n", + " comm.send({event_type: 'delete', 'id': id});\n", + " }\n", + " delete PyViz.plot_index[id];\n", + " if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n", + " var doc = window.Bokeh.index[id].model.document\n", + " doc.clear();\n", + " const i = window.Bokeh.documents.indexOf(doc);\n", + " if (i > -1) {\n", + " window.Bokeh.documents.splice(i, 1);\n", + " }\n", + " }\n", + "}\n", + "\n", + "/**\n", + " * Handle kernel restart event\n", + " */\n", + "function handle_kernel_cleanup(event, handle) {\n", + " delete PyViz.comms[\"hv-extension-comm\"];\n", + " window.PyViz.plot_index = {}\n", + "}\n", + "\n", + "/**\n", + " * Handle update_display_data messages\n", + " */\n", + "function handle_update_output(event, handle) {\n", + " handle_clear_output(event, {cell: {output_area: handle.output_area}})\n", + " handle_add_output(event, handle)\n", + "}\n", + "\n", + "function register_renderer(events, OutputArea) {\n", + " function append_mime(data, metadata, element) {\n", + " // create a DOM node to render to\n", + " var toinsert = this.create_output_subarea(\n", + " metadata,\n", + " CLASS_NAME,\n", + " EXEC_MIME_TYPE\n", + " );\n", + " this.keyboard_manager.register_events(toinsert);\n", + " // Render to node\n", + " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", + " render(props, toinsert[0]);\n", + " element.append(toinsert);\n", + " return toinsert\n", + " }\n", + "\n", + " events.on('output_added.OutputArea', handle_add_output);\n", + " events.on('output_updated.OutputArea', handle_update_output);\n", + " events.on('clear_output.CodeCell', handle_clear_output);\n", + " events.on('delete.Cell', handle_clear_output);\n", + " events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n", + "\n", + " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", + " safe: true,\n", + " index: 0\n", + " });\n", + "}\n", + "\n", + "if (window.Jupyter !== undefined) {\n", + " try {\n", + " var events = require('base/js/events');\n", + " var OutputArea = require('notebook/js/outputarea').OutputArea;\n", + " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", + " register_renderer(events, OutputArea);\n", + " }\n", + " } catch(err) {\n", + " }\n", + "}\n" + ], + "application/vnd.holoviews_load.v0+json": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n function JupyterCommManager() {\n }\n\n JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n comm_manager.register_target(comm_id, function(comm) {\n comm.on_msg(msg_handler);\n });\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n comm.onMsg = msg_handler;\n });\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n console.log(message)\n var content = {data: message.data, comm_id};\n var buffers = []\n for (var buffer of message.buffers || []) {\n buffers.push(new DataView(buffer))\n }\n var metadata = message.metadata || {};\n var msg = {content, buffers, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n })\n }\n }\n\n JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n if (comm_id in window.PyViz.comms) {\n return window.PyViz.comms[comm_id];\n } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n if (msg_handler) {\n comm.on_msg(msg_handler);\n }\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n comm.open();\n if (msg_handler) {\n comm.onMsg = msg_handler;\n }\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n var comm_promise = google.colab.kernel.comms.open(comm_id)\n comm_promise.then((comm) => {\n window.PyViz.comms[comm_id] = comm;\n if (msg_handler) {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n var content = {data: message.data};\n var metadata = message.metadata || {comm_id};\n var msg = {content, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n }\n }) \n var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n return comm_promise.then((comm) => {\n comm.send(data, metadata, buffers, disposeOnDone);\n });\n };\n var comm = {\n send: sendClosure\n };\n }\n window.PyViz.comms[comm_id] = comm;\n return comm;\n }\n window.PyViz.comm_manager = new JupyterCommManager();\n \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n var div = document.createElement(\"div\");\n var script = document.createElement(\"script\");\n node.appendChild(div);\n node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n var output_area = handle.output_area;\n var output = handle.output;\n if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n return\n }\n var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n if (id !== undefined) {\n var nchildren = toinsert.length;\n var html_node = toinsert[nchildren-1].children[0];\n html_node.innerHTML = output.data[HTML_MIME_TYPE];\n var scripts = [];\n var nodelist = html_node.querySelectorAll(\"script\");\n for (var i in nodelist) {\n if (nodelist.hasOwnProperty(i)) {\n scripts.push(nodelist[i])\n }\n }\n\n scripts.forEach( function (oldScript) {\n var newScript = document.createElement(\"script\");\n var attrs = [];\n var nodemap = oldScript.attributes;\n for (var j in nodemap) {\n if (nodemap.hasOwnProperty(j)) {\n attrs.push(nodemap[j])\n }\n }\n attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n oldScript.parentNode.replaceChild(newScript, oldScript);\n });\n if (JS_MIME_TYPE in output.data) {\n toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n }\n output_area._hv_plot_id = id;\n if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n window.PyViz.plot_index[id] = Bokeh.index[id];\n } else {\n window.PyViz.plot_index[id] = null;\n }\n } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n var bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n var script_attrs = bk_div.children[0].attributes;\n for (var i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n var id = handle.cell.output_area._hv_plot_id;\n var server_id = handle.cell.output_area._bokeh_server_id;\n if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n if (server_id !== null) {\n comm.send({event_type: 'server_delete', 'id': server_id});\n return;\n } else if (comm !== null) {\n comm.send({event_type: 'delete', 'id': id});\n }\n delete PyViz.plot_index[id];\n if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n var doc = window.Bokeh.index[id].model.document\n doc.clear();\n const i = window.Bokeh.documents.indexOf(doc);\n if (i > -1) {\n window.Bokeh.documents.splice(i, 1);\n }\n }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n delete PyViz.comms[\"hv-extension-comm\"];\n window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n handle_clear_output(event, {cell: {output_area: handle.output_area}})\n handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n var toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[0]);\n element.append(toinsert);\n return toinsert\n }\n\n events.on('output_added.OutputArea', handle_add_output);\n events.on('output_updated.OutputArea', handle_update_output);\n events.on('clear_output.CodeCell', handle_clear_output);\n events.on('delete.Cell', handle_clear_output);\n events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n safe: true,\n index: 0\n });\n}\n\nif (window.Jupyter !== undefined) {\n try {\n var events = require('base/js/events');\n var OutputArea = require('notebook/js/outputarea').OutputArea;\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n } catch(err) {\n }\n}\n" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import panel as pn\n", + "import param\n", + "from glob import glob\n", + "import base64\n", + "import re\n", + "import requests\n", + "from subprocess import Popen, PIPE, STDOUT\n", + "import html\n", + "from cairosvg import svg2png\n", + "import graphviz\n", + "from rdflib import Graph, Namespace, URIRef\n", + "import os\n", + "from collections import OrderedDict\n", + "from urllib.request import urlopen\n", + "import uuid\n", + "import shutil\n", + "\n", + "pn.extension(comms='ipywidgets')" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "7b4685d7-698d-4a86-a0a4-a81d337bc9d7", + "metadata": {}, + "outputs": [], + "source": [ + "#######################################################################################################\n", + "# Parameters\n", + "#######################################################################################################\n", + "shaclBase = '/opt/dashboards/tools/shacl-1.3.2/bin/'\n", + "owl2vowlPath = '/opt/dashboards/tools/owl2vowl_0.3.7/owl2vowl.jar'\n", + "storeBase = '../store/CCTP-SRSA-IP-20210831/'\n", + "extractionGraph = '/opt/dashboards/tetras-lab-unl-demos/work_graph.ttl' # -- old --- extractionGraph = '/opt/dashboards/tetras-lab-unl-demos/extraction-data-9.ttl'\n", + "workDir = 'work-data/'\n", + "webvowlData = '/opt/webvowl'\n", + "pane_width = 1300" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "2c41c319-4beb-4a85-a232-61a12d00cdbf", + "metadata": {}, + "outputs": [], + "source": [ + "#######################################################################################################\n", + "# UNL tools functions\n", + "#######################################################################################################\n", + "\n", + "def unl2stuff(unlFilePath, jarPath, outPrefix, outType):\n", + " # Run java parser\n", + " cmd = ['java', '-jar', jarPath,\n", + " '--input-file', unlFilePath,\n", + " '--output-file', outPrefix,\n", + " '--output-type', outType]\n", + " with Popen(cmd, stdout=PIPE, stderr=STDOUT) as p:\n", + " p.wait()\n", + " p.stdout.flush()\n", + " if p.returncode != 0:\n", + " print(\"Error in unl2rdf: \\n\\n\"+p.stdout.read().decode())\n", + " print('UNL;')\n", + " #print(text)\n", + " \n", + "def unl2dotWeb(unldata) :\n", + " data={'unl': unldata, 'outputs':['dot', 'svg', 'rdf']}\n", + " try:\n", + " r = requests.post('https://unl.demo.tetras-libre.fr/unl2rdf', data=data)\n", + " except Exception as e:\n", + " return 'Error calling https://unl.demo.tetras-libre.fr/unl2rdf : \"{error}\"'.format(error=e)\n", + " html=r.text\n", + " # On utilise une regex au lieu de parser le html car ce dernier est mal formé\n", + " regexSvg = re.compile('<svg.*svg>',re.MULTILINE|re.DOTALL)\n", + " regexRdf = re.compile(\"<code id='rdf' class='collapse show'>(.*?)</code>\",re.MULTILINE|re.DOTALL)\n", + " try : \n", + " svg = regexSvg.search(html).group()\n", + " rdf = regexRdf.search(html).group(1) \n", + " except Exception as e :\n", + " svg = ''\n", + " rdf = ''\n", + " print(e)\n", + " return(svg, rdf)\n", + "\n", + "def zipdir(path, ziph):\n", + " # ziph is zipfile handle\n", + " for root, dirs, files in os.walk(path):\n", + " for file in files:\n", + " if not('orig' in root):\n", + " ziph.write(os.path.join(root, file), \n", + " os.path.relpath(os.path.join(root, file), \n", + " os.path.join(path, '..')))\n", + "\n", + "def addBaseUri(rdfStr):\n", + " regexBaseUri = re.compile(\"http://rdf-unl.org.*?sentence.*?ontology\")\n", + " baseUri = regexBaseUri.search(rdfStr).group()\n", + " rdfStr = \"# baseURI: \"+baseUri+\"\\n\"+rdfStr\n", + " return(rdfStr)\n", + "\n", + "def postEditRdf(rdfPath, frStr, enStr):\n", + " textID = rdfPath.rsplit('/', 1)[0]\n", + " newPrefix = \"http://unsel.rdf-unl.org/\"+textID\n", + " with open(rdfPath,'r') as rdfFile :\n", + " rdfStr = rdfFile.read()\n", + " rdfFile.close()\n", + " regexBaseUri = re.compile(\"http://rdf-unl.org.*?sentence.*?ontology\")\n", + " rdfStr = rdfStr.replace('rdfs:label \"TBD : phrase en langue naturelle\"@inv ;', \n", + " '<https://unl.tetras-libre.fr/rdf/schema#has_id> \"{}\" ;\\n'.format(textID.split('/')[-2])+'rdfs:label \"\"\"{}\"\"\"@fr ;\\n'.format(frStr)+' rdfs:label \"\"\"{}\"\"\"@en ;\\n'.format(enStr))\n", + " baseUri = regexBaseUri.search(rdfStr).group()\n", + " oldPrefix = baseUri.rsplit('/', 1)[0]\n", + " rdfStr = rdfStr.replace(oldPrefix+'#ontology', newPrefix.rsplit('/', 1)[0]+'#ontology') \n", + " rdfStr = rdfStr.replace(oldPrefix+'#', \"http://unsel.rdf-unl.org/uw_lexeme#\")\n", + " rdfStr = \"# baseURI: \"+baseUri+\"\\n @prefix : <\"+baseUri.replace(\"ontology\",\"\")+\"> .\\n\"+rdfStr\n", + " rdfStr = rdfStr.replace(oldPrefix, newPrefix)\n", + " with open(rdfPath,'w') as rdfFile :\n", + " rdfStr = rdfFile.write(rdfStr)\n", + " rdfFile.close()\n", + "\n", + "def replaceInplace(filePath, searchText, replaceText):\n", + " #read input file\n", + " fin = open(filePath, \"rt\")\n", + " #read file contents to string\n", + " data = fin.read()\n", + " #replace all occurrences of the required string\n", + " data = data.replace(searchText, replaceText)\n", + " #close the input file\n", + " fin.close()\n", + " #open the input file in write mode\n", + " fin = open(filePath, \"wt\")\n", + " #overrite the input file with the resulting data\n", + " fin.write(data)\n", + " #close the file\n", + " fin.close()\n", + " \n", + "def createStoreDirectory(unlStr, srsaRef):\n", + " storeDir = storeBase+srsaRef+\"/current/\"\n", + " regexFr = re.compile(\"{org:fr}\\n(.*?)\\n{/org}\",re.MULTILINE|re.DOTALL)\n", + " try:\n", + " frStr = regexFr.search(unlStr).group(1)\n", + " except AttributeError:\n", + " frStr = '' \n", + " enStr = ''\n", + " # Create a directory named after 'Référence'\n", + " try :\n", + " os.makedirs(storeDir)\n", + " except FileExistsError:\n", + " pass\n", + " # Add english translation to unl code\n", + " unlStr = unlStr.replace(\"{/org}\", \"{{/org}}\\n{{en}}\\n{enStr}\\n{{/en}}\".format(enStr=enStr)) \n", + " # Write UNL code to a file\n", + " with open(storeDir+srsaRef+'.unl','w') as unlFile:\n", + " unlFile.write(unlStr)\n", + " os.chmod(storeDir+srsaRef+'.unl',0o766)\n", + " # Send UNL code to https://unl.demo.tetras-libre.fr/unl2rdf to get SVG and RDF\n", + " #svg, rdf = unl2dotWeb(unlStr)\n", + " \n", + " # Use unltools jar to create ttl and dot file from unl \n", + " unl2stuff(storeDir+srsaRef+'.unl', '/opt/dashboards/tools/unl2rdf-app-0.9.jar', storeDir+srsaRef, 'rdf')\n", + " postEditRdf(storeDir+srsaRef+'.ttl', frStr, enStr)\n", + " unl2stuff(storeDir+srsaRef+'.unl', '/opt/dashboards/tools/unl2rdf-app-0.9.jar', storeDir+srsaRef, 'dot')\n", + " \n", + " # Generate svg and png\n", + " graphviz.render('dot', 'svg', storeDir+srsaRef+'.dot') \n", + " graphviz.render('dot', 'png', storeDir+srsaRef+'.dot')\n", + " # Rename generated svg and png so they are not named like file.dot.svg\n", + " svgList = glob(storeDir+srsaRef+\"*.svg\")\n", + " for svgPath in svgList:\n", + " svgNewPath = svgPath.replace(\".dot\",\"\")\n", + " os.rename(svgPath, svgNewPath)\n", + " pngList = glob(storeDir+srsaRef+\"*.png\")\n", + " for pngPath in pngList:\n", + " pngNewPath = pngPath.replace(\".dot\",\"\")\n", + " os.rename(pngPath, pngNewPath)\n", + " \n", + " # Add full text sentences to the svg\n", + " replaceInplace(storeDir+srsaRef+'.svg', '</svg>','''<text x=\"0\" y=\"-40\">fr : {fr}</text>\n", + "<text x=\"0\" y=\"-20\">en : {en}</text>\n", + "</svg>'''.format(fr=frStr, en=enStr))\n", + " \n", + " #svgWithTxt = svg.replace('</svg>','''<text x=\"0\" y=\"-40\">fr : {fr}</text>\n", + "#<text x=\"0\" y=\"-20\">en : {en}</text>\n", + "#</svg>'''.format(fr=frStr, en=enStr)) \n", + " \n", + " #with open(storeDir+srsaRef+'.svg','w') as svgFile:\n", + " # svgFile.write(svgWithTxt)\n", + " #os.chmod(storeDir+srsaRef+'.svg',0o766)\n", + " #with open(storeDir+srsaRef+'.ttl','w') as rdfFile:\n", + " # rdfFile.write(rdf)\n", + " os.chmod(storeDir+srsaRef+'.ttl',0o766)\n", + " os.chmod(storeDir+srsaRef+'.svg',0o766)\n", + " os.chmod(storeDir+srsaRef+'.png',0o766)\n", + " os.chmod(storeDir+srsaRef+'.dot',0o766)\n", + " os.chmod(storeDir+srsaRef+'.unl',0o766)\n", + "\n", + " # Convert svg to png and write to a file\n", + " #try:\n", + " # svg2png(bytestring=svgWithTxt, write_to=storeDir+srsaRef+'.png')\n", + " #except :\n", + " # pass\n", + " shutil.copytree(storeDir, storeBase+srsaRef+\"/orig/\") \n", + " with open(storeBase+srsaRef+\"/current/\"+srsaRef+'.comments','w') as commentFile:\n", + " commentFile.write(\"[David] : Saisissez vos commentaires en commençant par votre nom, n'oubliez pas d'enregistrer : \")\n", + " os.chmod(storeBase+srsaRef+\"/current/\"+srsaRef+'.comments',0o766)\n", + "\n", + "def writeUnlFiles(unlStr, storePrefix):\n", + " srsaRef = selectDir.value\n", + " with open(storePrefix+'.unl','w') as unlFile:\n", + " unlFile.write(unlStr)\n", + " unlFile.close() \n", + " \n", + "def createFolderFromUnselInter(srsaRef):\n", + " url = 'https://lingwarium.org/UNseL-inter/GrapheUNL.txt'\n", + " unlStr = urlopen(url).read().decode('utf-8').replace('[P:1]','').replace('[/P]','')\n", + " createStoreDirectory(unlStr, srsaRef)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "7b32d69a-52fb-4b9d-8cd9-5fb45c177284", + "metadata": {}, + "outputs": [], + "source": [ + "#######################################################################################################\n", + "# Extraction\n", + "#######################################################################################################\n", + "\n", + "def run_command(cmd):\n", + " with Popen(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True) as p:\n", + " p.poll()\n", + " p.stdout.flush()\n", + " p.stderr.flush()\n", + " stdout, stderr = p.communicate()\n", + " return p.returncode, stdout, stderr\n", + "\n", + "# Run SHACL infer sh script. mode argument can take the values 'infer' or 'validate'\n", + "def shaclInfer(ttlPath, mode, ttlRulesPath = ''):\n", + " if ttlRulesPath == '':\n", + " cmd = ['sh', shaclBase+'/shacl'+mode+'.sh', '-datafile', ttlPath]\n", + " else:\n", + " cmd = ['sh', shaclBase+'/shacl'+mode+'.sh', '-datafile', ttlPath, '-shapesfile', ttlRulesPath]\n", + " #cmd = ' '.join(cmd) \n", + " #!{cmd}\n", + " code, out, err = run_command(cmd)\n", + " if code != 0:\n", + " print(\"Error in SHACL script: \\n\\n\"+err)\n", + " else:\n", + " return(out)\n", + " \n", + "def export_result(g):\n", + " export_file = 'output.ttl'\n", + " g.serialize(destination=export_file, base=base_uri, format='turtle')\n", + " \n", + "#shaclInfer('/opt/dashboards/tetras-lab-unl-demos/demo-cctp-40.ttl', 'infer')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "5c7164d7-c074-4aa3-9776-0cc5cc8f18f7", + "metadata": {}, + "outputs": [], + "source": [ + "#==============================================================================\n", + "# TENET: prepare work data\n", + "#------------------------------------------------------------------------------\n", + "# Prepare work data for extraction processing. \n", + "#==============================================================================\n", + "\n", + "#==============================================================================\n", + "# Parameters\n", + "#==============================================================================\n", + "\n", + "# Working directories\n", + "CONFIG_DIR = \"/opt/dashboards/tools/tenet/config/\"\n", + "FRAME_DIR = \"/opt/dashboards/tools/tenet/frame/\"\n", + "CORPUS_DIR = storeBase\n", + "OUTPUT_DIR = \"output/\"\n", + "\n", + "# Config Definition\n", + "TURTLE_SUFFIX = \".ttl\"\n", + "frame_file = \"system-ontology.ttl\"\n", + "dash_file = \"dash-data-shapes.ttl\" # data from \"http://datashapes.org/dash.ttl\"\n", + "schema_file = \"unl-rdf-schema.ttl\"\n", + "semantic_net_file = \"semantic-net.ttl\"\n", + "cts_file = \"transduction-schemes.ttl\"\n", + "c_param_file = \"config-parameters.ttl\"\n", + "\n", + "# Dev Tests\n", + "base_uri = \"https://unsel.tetras-libre.fr/tenet/working\"\n", + "\n", + "#==============================================================================\n", + "# Graph Initialization\n", + "#==============================================================================\n", + " \n", + "def load_config(work_graph): \n", + " file_ref = CONFIG_DIR + schema_file\n", + " work_graph.parse(file_ref)\n", + " \n", + " file_ref = CONFIG_DIR + semantic_net_file\n", + " work_graph.parse(file_ref)\n", + " \n", + " file_ref = CONFIG_DIR + dash_file\n", + " work_graph.parse(file_ref)\n", + " \n", + " file_ref = CONFIG_DIR + c_param_file\n", + " work_graph.parse(file_ref)\n", + "\n", + "def load_frame(work_graph):\n", + " file_ref = FRAME_DIR + frame_file\n", + " work_graph.parse(file_ref)\n", + "\n", + "#def define_namespace(work_graph):\n", + "# print(\"-- Namespace Definition:\")\n", + "# \n", + "# sys_uri = \"https://unsel.tetras-libre.fr/tenet/frame/system-ontology/\"\n", + "# concept_classes = [\"agent\"]\n", + "# for concept in concept_classes:\n", + "# new_prefix = \"sys-\" + concept\n", + "# new_uri = URIRef(sys_uri + concept + '#') \n", + "# work_graph.namespace_manager.bind(new_prefix, new_uri)\n", + "# print(\"----- \" + new_prefix + \": \" + new_uri)\n", + "# print(list(work_graph.namespace_manager.namespaces())) \n", + " \n", + "def load_sentences(work_graph, corpus):\n", + " target_ref = CORPUS_DIR + corpus + '/current/*.ttl'\n", + " for file_ref in glob(target_ref):\n", + " if 'factoid' not in file_ref :\n", + " work_graph.parse(file_ref)\n", + " \n", + "def load_cts(work_graph): \n", + " file_ref = CONFIG_DIR + cts_file\n", + " work_graph.parse(file_ref)\n", + " \n", + "#==============================================================================\n", + "# Result (export)\n", + "#==============================================================================\n", + " \n", + "def export_result(work_graph, export_ref, export_file): \n", + " work_graph.serialize(destination=export_file, \n", + " base=base_uri + '/' + export_ref, \n", + " format='turtle')\n", + " \n", + " \n", + "def finalize_export_file(export_file):\n", + " \"\"\" finalize the export file by adding some useful prefixes \"\"\"\n", + " \n", + " with open(export_file, \"rt\") as file:\n", + " x = file.read()\n", + " \n", + " with open(export_file, \"wt\") as file:\n", + " x = x.replace(\n", + " \"@prefix sys: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/> .\",\n", + " \"\"\"\n", + " @prefix sys: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/> .\n", + " @prefix sys-class: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/class/> .\n", + " @prefix sys-property: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/property/> .\n", + " @prefix sys-relation: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/relation/> .\n", + " @prefix sys-Event: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/Event#> .\n", + " @prefix sys-event: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/eventObjectProperty#> .\n", + " @prefix sys-State_Property: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/State_Property#> .\n", + " @prefix sys-stateProperty: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/statePropertyObjectProperty#> .\n", + " @prefix sys-abstract_thing: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/abstract_thing#> .\n", + " @prefix sys-action_verb: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/action_verb#> .\n", + " @prefix sys-agent: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/agent#> .\n", + " @prefix sys-attributive_verb: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/attributive_verb#> .\n", + " @prefix sys-component: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/component#> .\n", + " @prefix sys-message: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/message#> .\n", + " @prefix sys-place: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/place#> .\n", + " \"\"\")\n", + " file.write(x)\n", + " \n", + "#==============================================================================\n", + "# Main Function\n", + "#==============================================================================\n", + " \n", + "def createTenetGraph(corpus):\n", + " try: \n", + " work_graph = Graph()\n", + " load_config(work_graph) \n", + " load_frame(work_graph) \n", + " #define_namespace(work_graph)\n", + " load_cts(work_graph)\n", + " load_sentences(work_graph, corpus)\n", + " output_file = extractionGraph\n", + " export_result(work_graph, corpus, output_file)\n", + " finalize_export_file(output_file)\n", + " return(work_graph)\n", + " except Exception as e :\n", + " print(\"!!! An exception occurred importing rdf graphs for extraction !!!\\n\"+str(e)) \n", + " \n", + "def addSentenceInTenetGraph(work_graph, sentence_ref):\n", + " \"\"\" TODO: add a sentence to work_graph \"\"\"\n", + " pass\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f66bfcd2-f2b9-4603-b1f2-d4fb643c8c3c", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "clearExecutionInstances = \"\"\"\n", + " PREFIX cts: <https://unsel.tetras-libre.fr/tenet/transduction-schemes#>\n", + " PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>\n", + " PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>\n", + "\n", + " DELETE {?x rdf:type ?c}\n", + " WHERE {\n", + " ?c rdfs:subClassOf* cts:Transduction_Schemes .\n", + " ?x rdf:type ?c .\n", + " }\n", + "\"\"\"\n", + "\n", + "addExecutionInstance = \"\"\"\n", + " PREFIX cts: <https://unsel.tetras-libre.fr/tenet/transduction-schemes#>\n", + " PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>\n", + " PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>\n", + "\n", + " INSERT DATA {{<exec_instance> rdf:type {}}}\n", + "\"\"\"\n", + "\n", + "\n", + "\n", + "def owl2vowl(ttlFilePath, importList=[]):\n", + " # Run java parser\n", + " if importList == []:\n", + " cmd = ['java', '-jar', owl2vowlPath,\n", + " '-file', ttlFilePath] \n", + " else:\n", + " cmd = ['java', '-jar', owl2vowlPath,\n", + " '-file', ttlFilePath,\n", + " '-dependencies'] + importList \n", + " with Popen(cmd, stdout=PIPE, stderr=STDOUT) as p:\n", + " p.wait()\n", + " p.stdout.flush()\n", + " if p.returncode != 0:\n", + " print(\"Error in owl2vowl: \\n\\n\"+p.stdout.read().decode())\n", + " outFileName = ttlFilePath.split('/')[-1].replace('ttl','json')\n", + " os.rename(outFileName, '/opt/webvowl/'+outFileName)\n", + " \n", + "def applyInferStep(uuidStr, graph, step):\n", + " step_ref = \"cts:\" + step\n", + " dest_file = workDir + uuidStr + '-' + step + \".ttl\"\n", + " base_ref = \"http://\" + uuidStr + '/' + step\n", + " graph.update(clearExecutionInstances)\n", + " graph.update(addExecutionInstance.format(step_ref)) # ex. : step = 'cts:generation'\n", + " graph.serialize(destination=dest_file, base=base_ref, format='turtle') # serialize graph before inference\n", + " work_file = dest_file\n", + " inferResult = shaclInfer(work_file, 'infer') # apply SHACL inference\n", + " graph.parse(data=inferResult) # update graph with inference\n", + " graph.serialize(destination=dest_file, base=base_ref, format='turtle') # serialize graph after inference\n", + " return graph, inferResult\n", + " \n", + " \n", + "#owl2vowl('/opt/dashboards/printtatic/extraction.ttl', importList=['system-ontology.ttl'])\n", + "def createOnto(uuidStr):\n", + " \n", + " # -- Initialization (creation of extraction graph)\n", + " step = 'init_graph'\n", + " # -- old --- step_ref = \"cts:\" + step\n", + " dest_file = workDir + uuidStr + '-' + step + \".ttl\"\n", + " base_ref = \"http://\" + uuidStr + '/' + step\n", + " graph = createTenetGraph(uuidStr)\n", + " graph.serialize(destination=dest_file, base=base_ref, format='turtle')\n", + " \n", + " # -- Extraction\n", + " graph, _ = applyInferStep(uuidStr, graph, 'preprocessing')\n", + " graph, _ = applyInferStep(uuidStr, graph, 'net_extension')\n", + " graph, finalInferResult = applyInferStep(uuidStr, graph, 'generation_dga_patch')\n", + " # -- old --- graph, finalInferResult = applyInferStep(uuidStr, graph, 'generation')\n", + " \n", + " # -- dev --- Generation step by step\n", + " # -- old --- graph, _ = applyInferStep(uuidStr, graph, 'class_generation')\n", + " # -- old --- graph, _ = applyInferStep(uuidStr, graph, 'relation_generation_1')\n", + " # -- old --- graph, _ = applyInferStep(uuidStr, graph, 'relation_generation_2')\n", + " # -- old --- graph, _ = applyInferStep(uuidStr, graph, 'relation_generation_3_1')\n", + " # -- old --- graph, finalInferResult = applyInferStep(uuidStr, graph, 'relation_generation_3_2')\n", + " # -- old --- graph, finalInferResult = applyInferStep(uuidStr, graph, 'relation_generation')\n", + " # -- dev ---\n", + " \n", + " # -- Preprocessing\n", + " # -- old --- step = 'preprocessing'\n", + " # -- old --- step_ref = \"cts:\" + step\n", + " # -- old --- dest_file = step + \".ttl\"\n", + " # -- old --- base_ref = \"http://\" + step\n", + " # -- old --- graph.update(clearExecutionInstances)\n", + " # -- old --- graph.update(addExecutionInstance.format(step_ref)) # ex. : step = 'cts:generation'\n", + " # -- old --- graph.serialize(destination=dest_file, base=base_ref, format='turtle')\n", + " # -- old --- work_file = dest_file\n", + " # -- old --- inferResult1 = shaclInfer(work_file, 'infer')\n", + " # -- old --- graph.parse(data=inferResult1)\n", + " # -- old --- graph.serialize(destination=dest_file, base=base_ref, format='turtle')\n", + " \n", + " # -- Net Extension \n", + " # -- old --- step = 'net_extension'\n", + " # -- old --- step_ref = \"cts:\" + step\n", + " # -- old --- dest_file = step + \".ttl\"\n", + " # -- old --- base_ref = \"http://\" + step\n", + " # -- old --- graph.update(clearExecutionInstances)\n", + " # -- old --- graph.update(addExecutionInstance.format(step_ref)) # ex. : step = 'cts:generation'\n", + " # -- old --- graph.serialize(destination=dest_file, base=base_ref, format='turtle')\n", + " # -- old --- work_file = dest_file\n", + " # -- old --- inferResult2 = shaclInfer(work_file, 'infer')\n", + " # -- old --- graph.parse(data=inferResult2)\n", + " # -- old --- graph.serialize(destination=dest_file, base=base_ref, format='turtle')\n", + " \n", + " # -- Generation\n", + " # -- old --- step = 'generation'\n", + " # -- old --- step_ref = \"cts:\" + step\n", + " # -- old --- dest_file = step + \".ttl\"\n", + " # -- old --- base_ref = \"http://\" + step\n", + " # -- old --- graph.update(clearExecutionInstances)\n", + " # -- old --- graph.update(addExecutionInstance.format(step_ref)) # ex. : step = 'cts:generation'\n", + " # -- old --- graph.serialize(destination=dest_file, base=base_ref, format='turtle') # serialize graph before inference\n", + " # -- old --- work_file = dest_file\n", + " # -- old --- finalInferResult = shaclInfer(work_file, 'infer') # apply SHACL inference\n", + " # -- old --- graph.parse(data=finalInferResult) # update graph with inference\n", + " # -- old --- graph.serialize(destination=dest_file, base=base_ref, format='turtle') # serialize graph after inference\n", + " \n", + " # -- Result\n", + " # -- old --- file_ref = CONFIG_DIR + schema_file\n", + " # -- old --- sys_frame_onto = FRAME_DIR + frame_file\n", + " factoidPath = storeBase+uuidStr+'/current/'+uuidStr+'_factoid.ttl'\n", + " with open(factoidPath, 'w') as outfile:\n", + " outfile.write(finalInferResult)\n", + " outfile.close()\n", + " \n", + " owl2vowl(factoidPath)#, importList=[sys_frame_onto])\n", + " \n", + " #result_graph.parse(data=finalInferResult) # update with inferences resulting of extraction final step\n", + " #factoidPath = storeBase+uuidStr+'/current/'+uuidStr+'_factoid.ttl'\n", + " #result_graph.serialize(destination=factoidPath, base=base_ref, format='turtle') \n", + " \n", + " # On exécute ensuite seulement les règles de génération pour récupérer seulement le nécesaire à la visualisation\n", + " # -- old --- graph.update(clearExecutionInstances)\n", + " # -- old --- graph.update(addExecutionInstance.format('cts:generation'))\n", + " # -- old --- graph.serialize(destination=\"tmp.ttl\", base=\"http://tmp\", format='turtle')\n", + " \n", + " # -- old --- factoidPath = storeBase+uuidStr+'/current/'+uuidStr+'_factoid.ttl'\n", + " # -- old --- with open(factoidPath, 'w') as outfile:\n", + " # -- old --- inferResult3 = shaclInfer('tmp.ttl', 'infer')\n", + " # -- old --- outfile.write(inferResult3)\n", + " # -- old --- outfile.close()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "40b54849-9333-4819-b953-6e816ffe474c", + "metadata": {}, + "outputs": [], + "source": [ + "#######################################################################################################\n", + "# Validation\n", + "#######################################################################################################\n", + "def pyshaclValidate():\n", + " from pyshacl import validate\n", + " data_file = open('tmp.ttl').read()\n", + " shapes_file = open('test-shacl-construct.shapes-order.ttl').read()\n", + " conforms, v_graph, v_text = validate(data_file, shacl_graph=shapes_file)\n", + " print(conforms)\n", + " print(v_graph)\n", + " print(v_text)\n", + " \n", + "def loadFactoids(directory):\n", + " ttlList = glob(directory+\"/*/current/*.ttl\")\n", + " g = Graph()\n", + " for ttl in ttlList :\n", + " g.parse(ttl)\n", + " g.parse('/opt/dashboards/tools/tenet/frame/system-ontology.ttl')\n", + " g.parse('/opt/dashboards/tools/tenet/config/unl-rdf-schema.ttl')\n", + " return(g)\n", + " \n", + "possibleUnderspecificationQuery ='''\n", + "PREFIX owl: <http://www.w3.org/2002/07/owl#>\n", + "PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>\n", + "PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>\n", + "PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>\n", + "PREFIX unl: <https://unl.tetras-libre.fr/rdf/schema#>\n", + "PREFIX net: <https://unsel.tetras-libre.fr/tenet/semantic-net#>\n", + "PREFIX cprm: <https://unsel.tetras-libre.fr/tenet/config/parameters#>\n", + "PREFIX req: <https://unsel.tetras-libre.fr/tenet/frame/requirement-ontology#>\n", + "PREFIX sys: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/>\n", + "PREFIX fprm: <https://unsel.tetras-libre.fr/tenet/frame/parameters#>\n", + "\n", + "SELECT ?thisId (CONCAT('Parle-t-on bien de tous les \"', ?entityLabel, '\" possibles ? (', ?subEntities, ')' ) AS ?message)\n", + "WHERE {\n", + "FILTER(?count>1)\n", + "{SELECT ?this ?thisId ?entityLabel (GROUP_CONCAT(?subEntityLabel;SEPARATOR=\", \") AS ?subEntities) (COUNT(?subEntityLabel) AS ?count)\n", + "WHERE {\n", + "\t?subEntity rdfs:subClassOf ?entity ; rdfs:label ?subEntityLabel .\n", + "\t{SELECT ?this ?entity ?thisId ?entityLabel\n", + "\tWHERE {\n", + " BIND(\"##ID##\" AS ?thisId)\n", + " \t?this a unl:UNL_Sentence ; unl:has_id ?thisId .\n", + "\t\t?entity sys:from_structure ?this ; \n", + "\t\t\trdfs:subClassOf+ sys:Structure ;\n", + "\t\t\trdfs:label ?entityLabel .\n", + "\t\t FILTER (\n", + " \t\t!EXISTS {?subEntity1 rdfs:subClassOf ?entity; sys:from_structure ?this}\n", + " )\n", + "\t}} \n", + "}\n", + "GROUP BY ?this ?thisId ?entityLabel }\n", + "}\n", + "'''\n", + "\n", + "possibleClassEquivalenceQuery = '''PREFIX owl: <http://www.w3.org/2002/07/owl#>\n", + "PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>\n", + "PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>\n", + "PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>\n", + "PREFIX unl: <https://unl.tetras-libre.fr/rdf/schema#>\n", + "PREFIX net: <https://unsel.tetras-libre.fr/tenet/semantic-net#>\n", + "PREFIX cprm: <https://unsel.tetras-libre.fr/tenet/config/parameters#>\n", + "PREFIX req: <https://unsel.tetras-libre.fr/tenet/frame/requirement-ontology#>\n", + "PREFIX sys: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/>\n", + "PREFIX fprm: <https://unsel.tetras-libre.fr/tenet/frame/parameters#>\n", + "\n", + "SELECT (CONCAT(?messageTMP, ?sentenceList, ')') AS ?message) ?sentenceList\n", + "WHERE {\n", + "SELECT ?messageTMP (GROUP_CONCAT(?sentenceId; SEPARATOR=\", \") AS ?sentenceList)\n", + "WHERE {\t\n", + "\tSELECT DISTINCT ?messageTMP ?sentenceId\n", + "\tWHERE {\n", + "\t\tFILTER (?count = 1)\n", + "\t\t?subEntity rdfs:subClassOf ?this ; rdfs:label ?subEntityLabel ; sys:from_structure ?subEntitySentence .\n", + "\t\t?this rdfs:label ?thisLabel ; sys:from_structure ?thisSentence .\n", + " \tBIND(CONCAT('\"', ?subEntityLabel, '\" est la seule sous classe de \"', ?thisLabel, '\". Ces classes sont-elles équivalentes ? <br/>(voir les exigences ') AS ?messageTMP)\n", + " \t\t{BIND(\"##ID##\" AS ?thisSentenceId)\n", + " ?thisSentence unl:has_id ?thisSentenceId .\n", + "\t\t\tBIND (?thisSentenceId AS ?sentenceId)} \n", + "\t\tUNION \n", + "\t\t\t{?subEntitySentence unl:has_id ?subEntitySentenceId .\n", + "\t\t\tBIND (?subEntitySentenceId AS ?sentenceId)}\n", + "\tFILTER(NOT EXISTS {?subEntity sys:from_structure ?thisSentence})\n", + "\t\t{SELECT ?this (COUNT(?subClass) AS ?count)\n", + "\t\tWHERE {\n", + " \t \t?this rdfs:subClassOf+ sys:Structure .\n", + "\t\t\t\t?subClass rdfs:subClassOf ?this\n", + "\t\t} GROUP BY ?this } \n", + "\t} ORDER BY ?sentenceId\n", + "} GROUP BY ?messageTMP\n", + "}'''\n", + "\n", + "unfeaturedDomainOrRangeQuery = '''PREFIX owl: <http://www.w3.org/2002/07/owl#>\n", + "PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>\n", + "PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>\n", + "PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>\n", + "PREFIX unl: <https://unl.tetras-libre.fr/rdf/schema#>\n", + "PREFIX net: <https://unsel.tetras-libre.fr/tenet/semantic-net#>\n", + "PREFIX cprm: <https://unsel.tetras-libre.fr/tenet/config/parameters#>\n", + "PREFIX req: <https://unsel.tetras-libre.fr/tenet/frame/requirement-ontology#>\n", + "PREFIX sys: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/>\n", + "PREFIX fprm: <https://unsel.tetras-libre.fr/tenet/frame/parameters#>\n", + "\n", + "SELECT ?sentenceId (CONCAT(?messageTMP, GROUP_CONCAT(?featureLabel ; SEPARATOR=', ')) AS ?message)\n", + "WHERE {\n", + " SELECT DISTINCT ?sentenceId ?featureLabel (CONCAT( 'Dans cette exigence, \"', ?unfeaturedLabel, '\" pourrait être précisé par : ') AS ?messageTMP)\n", + " WHERE {\n", + " {\n", + " ?p rdfs:subPropertyOf+ sys:objectProperty ;\n", + " rdfs:domain ?featured ; \n", + " rdfs:range ?unfeatured .\n", + " }\n", + " UNION\n", + " {\n", + " ?p rdfs:subPropertyOf+ sys:objectProperty ;\n", + " rdfs:domain ?unfeatured ; \n", + " rdfs:range ?featured .\n", + " }\n", + " ?p rdfs:label ?pLabel .\n", + " ?featured sys:has_feature ?feature .\n", + " FILTER(NOT EXISTS {\n", + " ?unfeatured sys:has_feature ?feature\n", + " })\n", + " ?featuredInstance a ?featured ; sys:from_structure ?sentence.\n", + " ?unfeaturedInstance a ?unfeatured ; sys:from_structure ?sentence.\n", + " BIND(\"##ID##\" AS ?sentenceId)\n", + " ?sentence unl:has_id ?sentenceId .\n", + " FILTER(NOT EXISTS {\n", + " ?featuredInstance a ?featured2 .\n", + " ?featured2 rdfs:subClassOf ?featured .\n", + " })\n", + " FILTER(NOT EXISTS {\n", + " ?unfeaturedInstance a ?unfeatured2 .\n", + " ?unfeatured2 rdfs:subClassOf ?unfeatured .\n", + " })\n", + " ?featured rdfs:label ?featuredLabel .\n", + " ?unfeatured rdfs:label ?unfeaturedLabel .\n", + " ?feature rdfs:label ?featureLabel .\n", + "}\n", + " } GROUP BY ?sentenceId ?messageTMP\n", + "'''\n", + "\n", + "unfeaturedDomainOrRangeWithRefQuery ='''PREFIX owl: <http://www.w3.org/2002/07/owl#>\n", + "PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>\n", + "PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>\n", + "PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>\n", + "PREFIX unl: <https://unl.tetras-libre.fr/rdf/schema#>\n", + "PREFIX net: <https://unsel.tetras-libre.fr/tenet/semantic-net#>\n", + "PREFIX cprm: <https://unsel.tetras-libre.fr/tenet/config/parameters#>\n", + "PREFIX req: <https://unsel.tetras-libre.fr/tenet/frame/requirement-ontology#>\n", + "PREFIX sys: <https://unsel.tetras-libre.fr/tenet/frame/system-ontology/>\n", + "PREFIX fprm: <https://unsel.tetras-libre.fr/tenet/frame/parameters#>\n", + "\n", + "SELECT ?sentenceId (CONCAT('\"', ?unfeaturedLabel, '\" pourrait être précisé par un ou plusieurs attributs parmis : ', ?featureList, '. <br/>(exemples de référence : ', GROUP_CONCAT(?sentence2id ; SEPARATOR=', '), ').') AS ?message) \n", + "WHERE {\n", + "{SELECT DISTINCT ?sentenceId ?unfeaturedLabel ?sentence2id (GROUP_CONCAT(?featureLabel ; SEPARATOR=', ') AS ?featureList) #\n", + "WHERE {\n", + " SELECT DISTINCT ?sentenceId ?sentence2id ?unfeaturedLabel ?featureLabel ?otherwiseFeaturedLabel ?featured2label\n", + " WHERE {\n", + " {\n", + " ?p rdfs:subPropertyOf+ sys:objectProperty ;\n", + " rdfs:domain ?featured ; \n", + " rdfs:range ?unfeatured .\n", + " }\n", + " UNION\n", + " {\n", + " ?p rdfs:subPropertyOf+ sys:objectProperty ;\n", + " rdfs:domain ?unfeatured ; \n", + " rdfs:range ?featured .\n", + " }\n", + " ?p rdfs:label ?pLabel .\n", + " ?featured sys:has_feature ?feature .\n", + " FILTER(NOT EXISTS {\n", + " ?unfeatured sys:has_feature ?feature\n", + " })\n", + " ?featuredInstance a ?featured ; sys:from_structure ?sentence.\n", + " ?unfeaturedInstance a ?unfeatured ; sys:from_structure ?sentence.\n", + " BIND(\"##ID##\" AS ?sentenceId)\n", + " ?sentence unl:has_id ?sentenceId .\n", + "\n", + "\t?otherwiseFeatured rdfs:subClassOf ?unfeatured ; sys:has_feature ?feature2 ; rdfs:label ?otherwiseFeaturedLabel.\n", + " ?otherwiseFeaturedInstance a ?otherwiseFeatured ; sys:from_structure ?sentence2.\t\n", + "\t?sentence2 unl:has_id ?sentence2id .\n", + "\t{?otherwiseFeaturedInstance ?p2 ?featuredInstance2} UNION { ?featuredInstance2 ?p2 ?otherwiseFeaturedInstance}\n", + "\t?featuredInstance2 a ?featured2 .\n", + "\t?featured2 sys:has_feature ?feature2 ; rdfs:label ?featured2label.\n", + "\t\t\n", + " FILTER(NOT EXISTS {\n", + " ?featuredInstance a ?featured2 .\n", + " ?featured2 rdfs:subClassOf ?featured .\n", + " })\n", + " FILTER(NOT EXISTS {\n", + " ?unfeaturedInstance a ?unfeatured2 .\n", + " ?unfeatured2 rdfs:subClassOf ?unfeatured .\n", + " })\n", + " ?featured rdfs:label ?featuredLabel .\n", + " ?unfeatured rdfs:label ?unfeaturedLabel .\n", + " ?feature rdfs:label ?featureLabel .\n", + " }\n", + "} GROUP BY ?sentenceId ?unfeaturedLabel ?sentence2id}\n", + "} GROUP BY ?sentenceId ?unfeaturedLabel ?featureList\n", + "'''\n", + "\n", + "queryTypeDic = {'error':[possibleClassEquivalenceQuery], \n", + " 'warning':[possibleUnderspecificationQuery], \n", + " 'info':[unfeaturedDomainOrRangeQuery]}\n", + "\n", + "# On charge les factoïdes pour vérification des nouvelles phrases\n", + "g = loadFactoids(storeBase)\n", + "#g.serialize(destination='/opt/dashboards/store/SRSA-IP_demo.ttl', format='turtle')\n", + "\n", + "errorReqDic = {}\n", + "warningReqDic = {}\n", + "infoReqDic = {}\n", + "\n", + "\n", + "#possibleUnderspecification = g.query(possibleUnderspecificationQuery)\n", + "#possibleClassEquivalence = g.query(possibleClassEquivalenceQuery)\n", + "#unfeaturedDomainOrRange = g.query(unfeaturedDomainOrRangeQuery)\n", + "\n", + "\n", + "\n", + "#for r in possibleUnderspecification : \n", + "# if str(r['thisId']) in warningReqDic:\n", + "# warningReqDic[str(r['thisId'])] += [str(r['message'])]\n", + "# else:\n", + "# warningReqDic[str(r['thisId'])] = [str(r['message'])] \n", + "\n", + "#for r in possibleClassEquivalence : \n", + "# sentenceList = str(r['sentenceList']).split(', ')\n", + "# for sentence in sentenceList:\n", + "# if sentence in errorReqDic:\n", + "# errorReqDic[sentence] += [str(r['message'])]\n", + "# else:\n", + "# errorReqDic[sentence] = [str(r['message'])] \n", + "\n", + "#for r in unfeaturedDomainOrRange : \n", + "# if str(r['sentenceId']) in infoReqDic:\n", + "# infoReqDic[str(r['sentenceId'])] += [str(r['message'])]\n", + "# else:\n", + "# infoReqDic[str(r['sentenceId'])] = [str(r['message'])] \n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "744abdb9-b3d6-4025-abc9-2f749644c3ed", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/opt/dashboards/store/CCTP-SRSA-IP-20210831/SRSA-IP_STB_PHON_01100\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/tmp/ipykernel_29142/776154484.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[0moutFile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m \u001b[0mupdateAllVerificationMessages\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 36\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/tmp/ipykernel_29142/776154484.py\u001b[0m in \u001b[0;36mupdateAllVerificationMessages\u001b[0;34m()\u001b[0m\n\u001b[1;32m 23\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mqueryTMP\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mqueryTypeDic\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmessageType\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0mquery\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mqueryTMP\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"##ID##\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mreqId\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 26\u001b[0m \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'message'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0;31m#print(message)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/query.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 290\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 291\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_genbindings\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 292\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_genbindings\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 293\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# don't add a result row in case of empty binding {}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 294\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_bindings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/evaluate.py\u001b[0m in \u001b[0;36m<genexpr>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 508\u001b[0m \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevalPart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 509\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 510\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mproject\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPV\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mrow\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 511\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 512\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/evaluate.py\u001b[0m in \u001b[0;36mevalExtend\u001b[0;34m(ctx, extend)\u001b[0m\n\u001b[1;32m 91\u001b[0m \u001b[0;31m# TODO: Deal with dict returned from evalPart from GROUP BY\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mevalPart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mextend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 94\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[0me\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_eval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_except\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_vars\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/evaluate.py\u001b[0m in \u001b[0;36mevalExtend\u001b[0;34m(ctx, extend)\u001b[0m\n\u001b[1;32m 91\u001b[0m \u001b[0;31m# TODO: Deal with dict returned from evalPart from GROUP BY\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mevalPart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mextend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 94\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[0me\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_eval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_except\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_vars\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/evaluate.py\u001b[0m in \u001b[0;36mevalAggregateJoin\u001b[0;34m(ctx, agg)\u001b[0m\n\u001b[1;32m 419\u001b[0m \u001b[0maggregator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 420\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 421\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mrow\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 422\u001b[0m \u001b[0;31m# determine right group aggregator for row\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 423\u001b[0m \u001b[0mk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_eval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0me\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgroup_expr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/evaluate.py\u001b[0m in \u001b[0;36mevalDistinct\u001b[0;34m(ctx, part)\u001b[0m\n\u001b[1;32m 499\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 500\u001b[0m \u001b[0mdone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 501\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 502\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 503\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/evaluate.py\u001b[0m in \u001b[0;36m<genexpr>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 508\u001b[0m \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevalPart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 509\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 510\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mproject\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPV\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mrow\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 511\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 512\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/evaluate.py\u001b[0m in \u001b[0;36mevalExtend\u001b[0;34m(ctx, extend)\u001b[0m\n\u001b[1;32m 91\u001b[0m \u001b[0;31m# TODO: Deal with dict returned from evalPart from GROUP BY\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mevalPart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mextend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 94\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[0me\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_eval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_except\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_vars\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/evaluate.py\u001b[0m in \u001b[0;36mevalFilter\u001b[0;34m(ctx, part)\u001b[0m\n\u001b[1;32m 170\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mevalFilter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpart\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 171\u001b[0m \u001b[0;31m# TODO: Deal with dict returned from evalPart!\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 172\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mevalPart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpart\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 173\u001b[0m if _ebv(\n\u001b[1;32m 174\u001b[0m \u001b[0mpart\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpr\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/evaluate.py\u001b[0m in \u001b[0;36mevalPart\u001b[0;34m(ctx, part)\u001b[0m\n\u001b[1;32m 252\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mevalFilter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpart\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 253\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mpart\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"Join\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 254\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mevalJoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpart\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 255\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mpart\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"LeftJoin\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 256\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mevalLeftJoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpart\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/evaluate.py\u001b[0m in \u001b[0;36mevalJoin\u001b[0;34m(ctx, join)\u001b[0m\n\u001b[1;32m 125\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevalPart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjoin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mp1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 127\u001b[0;31m \u001b[0mb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevalPart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjoin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mp2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 128\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_join\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/evaluate.py\u001b[0m in \u001b[0;36mevalBGP\u001b[0;34m(ctx, bgp)\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 85\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mevalBGP\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbgp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 87\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/evaluate.py\u001b[0m in \u001b[0;36mevalBGP\u001b[0;34m(ctx, bgp)\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 85\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mevalBGP\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbgp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 87\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/evaluate.py\u001b[0m in \u001b[0;36mevalBGP\u001b[0;34m(ctx, bgp)\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 85\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mevalBGP\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbgp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 87\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/evaluate.py\u001b[0m in \u001b[0;36mevalBGP\u001b[0;34m(ctx, bgp)\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 85\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mevalBGP\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbgp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 87\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/evaluate.py\u001b[0m in \u001b[0;36mevalBGP\u001b[0;34m(ctx, bgp)\u001b[0m\n\u001b[1;32m 54\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mbgp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m \u001b[0;32myield\u001b[0m \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msolution\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 57\u001b[0m \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/sparql.py\u001b[0m in \u001b[0;36msolution\u001b[0;34m(self, vars)\u001b[0m\n\u001b[1;32m 333\u001b[0m )\n\u001b[1;32m 334\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 335\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mFrozenBindings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbindings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 336\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 337\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__setitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/sparql.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, ctx, *args, **kwargs)\u001b[0m\n\u001b[1;32m 158\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mFrozenBindings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mFrozenDict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 159\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 160\u001b[0;31m \u001b[0mFrozenDict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 161\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mctx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mctx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 162\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/site-packages/rdflib/plugins/sparql/sparql.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 105\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_d\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 106\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_hash\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 107\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.8/_collections_abc.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 741\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 742\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__iter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 743\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mapping\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 744\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mapping\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 745\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "# Fonctions pour la mise à jour globale du corpus (ne pas exécuter en mode \"Tableau de bord\")\n", + "\n", + "def updateAllFactoids():\n", + " dirList = glob('/opt/dashboards/store/CCTP-SRSA-IP-20210831/*')\n", + " for directory in dirList:\n", + " if directory.split('/')[-1] != '0_NONE':\n", + " print(directory)\n", + " reqId = directory.split('/')[-1]\n", + " createOnto(reqId)\n", + "\n", + "#updateAllFactoids()\n", + " \n", + " \n", + "def updateAllVerificationMessages():\n", + " dirList = glob('/opt/dashboards/store/CCTP-SRSA-IP-20210831/*')\n", + " for directory in dirList:\n", + " if directory.split('/')[-1] != '0_NONE':\n", + " print(directory)\n", + " reqId = directory.split('/')[-1]\n", + " for messageType in ['error', 'warning', 'info']:\n", + " #print(messageType)\n", + " messagesStr = ''\n", + " for queryTMP in queryTypeDic[messageType] :\n", + " query = queryTMP.replace(\"##ID##\",reqId)\n", + " for r in g.query(query):\n", + " message = r['message']\n", + " #print(message)\n", + " if message not in messagesStr :\n", + " #print(\"printing\")\n", + " messagesStr+=message+\"\\n\"\n", + " outFile = open(directory+'/current/'+reqId+'_'+messageType+'Messages.txt', 'w')\n", + " outFile.write(messagesStr)\n", + " outFile.close() \n", + "\n", + "#updateAllVerificationMessages()\n", + "\n", + " \n", + "#query = possibleUnderspecificationQuery.replace(\"##ID##\",\"SRSA-IP_STB_PHON_00500\")\n", + "\n", + "#for r in g.query(query):\n", + " #print(r['message'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ca43f1f2-42ef-4355-a2e2-e27351a51b96", + "metadata": {}, + "outputs": [], + "source": [ + "#######################################################################################################\n", + "# Navigateur / éditeur de corpus UNL\n", + "#######################################################################################################\n", + "\n", + "saveButtonClicks = 0\n", + "\n", + "def main_pane(directory): \n", + " saveButtonClicks = 0\n", + " saveButton = pn.widgets.Button(name='Enregistrer', button_type='success', width = 100)\n", + " saveButtonDic = dict(button=saveButton)\n", + " saveCommentButton = pn.widgets.Button(name='Enregistrer', button_type='success', width = 100)\n", + " \n", + " path = storeBase+directory+'/current/'\n", + " pathOrig = storeBase+directory+'/orig/'\n", + " svgPath = path+directory+'.svg'\n", + " pngPath = path+directory+'.png'\n", + " unlPath = path+directory+'.unl'\n", + " rdfPath = path+directory+'.ttl'\n", + " commentPath = path+directory+'.comments'\n", + " with open(commentPath) as commentFile:\n", + " commentStr = commentFile.read() \n", + " commentFile.close()\n", + " with open(unlPath) as unlFile:\n", + " unlStr = unlFile.read()\n", + " unlFile.close()\n", + " svgPathOrig = pathOrig+directory+'.svg'\n", + " pngPathOrig = pathOrig+directory+'.png'\n", + " unlPathOrig = pathOrig+directory+'.unl'\n", + " rdfPathOrig = pathOrig+directory+'.ttl'\n", + " with open(unlPathOrig) as unlFileOrig:\n", + " unlStrOrig = unlFileOrig.read()\n", + " unlFileOrig.close()\n", + " unlHtmlOrig = unlStrOrig.replace(\"\\n\",\"<br/>\")\n", + " if unlStrOrig == unlStr:\n", + " modIndicator = ''\n", + " else:\n", + " modIndicator = ' <u>modifié</u>'\n", + " regexFr = re.compile(\"{org:fr}\\n(.*?)\\n{/org}\",re.MULTILINE|re.DOTALL)\n", + " try:\n", + " frStr = regexFr.search(unlStr).group(1)\n", + " except AttributeError:\n", + " frStr = ''\n", + " regexEn = re.compile(\"{en}\\n(.*?)\\n{/en}\",re.MULTILINE|re.DOTALL)\n", + " try:\n", + " enStr = regexEn.search(unlStr).group(1) \n", + " except AttributeError:\n", + " enStr = ''\n", + " \n", + " unlOrig_html = pn.pane.HTML(unlHtmlOrig)\n", + " unl_input = pn.widgets.input.TextAreaInput(height=400)\n", + " unl_input.value = unlStr\n", + " comment_input = pn.widgets.input.TextAreaInput(height=300)\n", + " comment_input.value = commentStr\n", + " \n", + " downloadSvg = pn.widgets.FileDownload(sizing_mode='stretch_width', file=svgPath, embed=True, name='Télécharger le graphe en SVG :')\n", + " downloadPng = pn.widgets.FileDownload(sizing_mode='stretch_width', file=pngPath, embed=True, name='Télécharger le graphe en PNG :') \n", + " downloadRdf = pn.widgets.FileDownload(sizing_mode='stretch_width', file=rdfPath, embed=True, name='Télécharger le code UNL-RDF :')\n", + " downloadUnl = pn.widgets.FileDownload(sizing_mode='stretch_width', file=unlPath, embed=True, name='Télécharger le code UNL :')\n", + " \n", + " def compute_unl_graph_pane(button):\n", + " global saveButtonClicks\n", + " if saveButtonClicks != 0:\n", + " writeUnlFiles(unl_input.value, storeBase+selectDir.value+'/current/'+selectDir.value)\n", + " pane = pn.pane.PNG(pngPath, width = pane_width)\n", + " saveButtonClicks += 1\n", + " return(pane)\n", + " \n", + " unl_graph_pane = pn.interact(lambda button : compute_unl_graph_pane(button), **saveButtonDic)\n", + " \n", + " \n", + " \n", + " warnColumn = pn.Card(width=pane_width, header = \"**Alertes automatiques pour l'exigence :**\")\n", + " warnings = 0\n", + " try: \n", + " errorfile = open(storeBase+directory+'/current/'+directory+'_errorMessages.txt','r')\n", + " errorlist = errorfile.readlines()\n", + " errorfile.close()\n", + " for error in errorlist: \n", + " warnColumn.append(pn.pane.HTML('<div class=\"alert alert-danger\">{}</div>'.format(error)))\n", + " if errorlist != [] : \n", + " warnings = 1\n", + " except :\n", + " pass\n", + " try: \n", + " warnfile = open(storeBase+directory+'/current/'+directory+'_warningMessages.txt','r')\n", + " warnlist = warnfile.readlines()\n", + " warnfile.close()\n", + " for warn in warnlist: \n", + " warnColumn.append(pn.pane.HTML('<div class=\"alert alert-warning\">{}</div>'.format(warn)))\n", + " if warnlist != [] : \n", + " warnings = 1\n", + " except :\n", + " pass \n", + " try: \n", + " infofile = open(storeBase+directory+'/current/'+directory+'_infoMessages.txt','r')\n", + " infolist = infofile.readlines()\n", + " infofile.close()\n", + " for info in infolist: \n", + " warnColumn.append(pn.pane.HTML('<div class=\"alert alert-info\">{}</div>'.format(info)))\n", + " if infolist != [] : \n", + " warnings = 1\n", + " except :\n", + " pass\n", + " if warnings == 0:\n", + " warnColumn.append(pn.pane.HTML('<div class=\"alert alert-info\">Pas d\\'anomalie détectée</div>'))\n", + " \n", + " pane = pn.Column(\n", + " pn.Row(\n", + " pn.pane.HTML('<a href=\"https://unsel.tetras-lab.io/webvowl#{}_factoid\" target=\"_blank\"><button type=\"button\" class=\"btn btn-outline-secondary btn-sm\">Visualiser l\\'ontologie construite</button><a>'.format(directory)),\n", + " pn.Column(pn.pane.HTML('<font size=\"tiny\">Exigence sélectionnée : '+directory+'</font>'), sizing_mode='stretch_width'), \n", + " ),\n", + " #pn.Card(\n", + " # pn.pane.HTML('''<iframe id=\"inlineFrameExample\"\n", + " # title=\"Inline Frame Example\"\n", + " # width=\"{}\"\n", + " # height=\"800\"\n", + " # src=\"https://unsel.tetras-lab.io/webvowl/#{}_factoid\">\n", + " # </iframe>'''.format(pane_width,selectDir.value)),\n", + " # title = \"Visualiser le factoid\", width=pane_width, collapsed=True),\n", + " warnColumn,\n", + " pn.pane.HTML('FR : '+frStr),\n", + " pn.pane.HTML('EN : '+enStr),\n", + " unl_graph_pane[1],\n", + " pn.Card(pn.Column(saveCommentButton, comment_input, width = pane_width),\n", + " header='**Commentaires**', \n", + " collapsed=True, width = pane_width), \n", + " pn.Card(pn.Column(saveButton, unl_input, width = pane_width),\n", + " header='**Code UNL**'+modIndicator, \n", + " collapsed=True, width = pane_width),\n", + " pn.Card(pn.Column(\n", + " unlOrig_html,\n", + " pn.Card(pn.pane.PNG(pngPath, width = pane_width-20), header = \"**Graphe d'origine**\", width=pane_width-10)\n", + " ), \n", + " header=\"**Code UNL d'origine**\", \n", + " collapsed=True, width = pane_width),\n", + " pn.WidgetBox(\n", + " pn.Row(downloadPng, downloadSvg),\n", + " pn.Row(downloadUnl, downloadRdf),\n", + " width=pane_width,\n", + " ),\n", + " width=pane_width, \n", + " )\n", + " \n", + " def saveComments(event):\n", + " with open(commentPath, 'w') as commentFile:\n", + " commentFile.write(comment_input.value) \n", + " commentFile.close()\n", + " saveCommentButton.on_click(saveComments) \n", + " return(pane)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d4ec56e-d0bb-44c8-975b-49d409b6b160", + "metadata": {}, + "outputs": [], + "source": [ + "#######################################################################################################\n", + "# Analyse avec UNseL-inter\n", + "#######################################################################################################\n", + "def extractOnClick(event):\n", + " uuidStr = \"COMP\"+str(uuid.uuid4())\n", + " createFolderFromUnselInter(uuidStr)\n", + " selectDir.options[uuidStr] = uuidStr\n", + " selectDir.value = uuidStr\n", + " createOnto(uuidStr)\n", + "\n", + "buttonExtract = pn.widgets.Button(name=\"Construire l'ontologie de la phrase (factoïd)\", width=300)\n", + "buttonExtract.param.watch(extractOnClick, 'clicks')\n", + "\n", + "#######################################################################################################\n", + "# Navigation Interface\n", + "#######################################################################################################\n", + "\n", + "pathList = glob(storeBase+'*')\n", + "dirList = sorted([x.split('/')[-1] for x in pathList])\n", + "warningList = sorted(list(errorReqDic.keys())+list(warningReqDic.keys())+list(infoReqDic.keys()))\n", + "\n", + "dirDic = {}\n", + "for directory in dirList:\n", + " if directory in warningList:\n", + " dirDic['**'+directory] = directory\n", + " else:\n", + " dirDic[directory] = directory\n", + "\n", + "dirDic = dict(sorted(dirDic.items()))\n", + " \n", + "selectDir = pn.widgets.Select(name='Sélectionnez une exigence : ', options=dirDic, width = 350)\n", + "selectDir.value = '0_NONE'\n", + "dir_selector = dict(directory=selectDir)#, save=saveButton)\n", + "\n", + "unlNavigatorPane = pn.interact(lambda directory : main_pane(directory), **dir_selector)\n", + "\n", + "#unl_graph_pane = pn.interact(lambda button : compute_unl_graph_pane(button), **saveButtonDic)\n", + "\n", + "# Main interface\n", + "pn.Column(\n", + "pn.Card( pn.Row(\n", + " pn.Spacer(sizing_mode='stretch_width'),\n", + " pn.Column(\n", + " pn.pane.HTML('''<iframe id=\"inlineFrameExample\"\n", + " title=\"Inline Frame Example\"\n", + " width=\"1000\"\n", + " height=\"600\"\n", + " src=\"https://lingwarium.org/UNseL-inter/\">\n", + " </iframe>'''),\n", + " buttonExtract),\n", + " pn.Spacer(sizing_mode='stretch_width'),\n", + " ), \n", + " title = \"Analyser une nouvelle exigence\", width=pane_width+50, collapsed=True,),\n", + " pn.layout.Divider(),\n", + " pn.Card( \n", + " pn.Row(\n", + " pn.layout.HSpacer(),\n", + " pn.Column(\n", + " selectDir,\n", + " unlNavigatorPane[1],\n", + " ),\n", + " pn.layout.HSpacer(),\n", + " ),\n", + " title = \"Naviguer dans les exigences\", width=pane_width+50, collapsed=True\n", + " ),\n", + " pn.layout.Divider(), \n", + " pn.Card(\n", + " pn.pane.HTML('''<iframe id=\"inlineFrameExample\"\n", + " title=\"Inline Frame Example\"\n", + " width=\"{}\"\n", + " height=\"800\"\n", + " src=\"https://unsel.tetras-lab.io/webvowl/#extraction\">\n", + " </iframe>'''.format(pane_width)),\n", + " pn.Row(\n", + " #pn.pane.HTML('<a href=\"https://unsel.tetras-lab.io/webvowl#extraction\" target=\"_blank\"><button type=\"button\" class=\"btn btn-outline-secondary btn-sm\">Visualiser l\\'ontologie construite pour tout le corpus</button><a>'),\n", + " #pn.pane.HTML('<a href=\"https://unsel.tetras-lab.io/static/webvowl_1.1.7/index.html#extraction_SRSA-IP_STB_PHON_00100\" target=\"_blank\"><button type=\"button\" class=\"btn btn-outline-secondary btn-sm\">Visualiser l\\'extraction de SRSA-IP_STB_PHON_00100</button><a>'), \n", + " pn.widgets.FileDownload(file='/opt/dashboards/static/extraction.ttl', embed=True)\n", + " ), title = \"Résultats sur le corpus\", width=pane_width+50, collapsed=True\n", + " ), \n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e06f5381-6c2d-4762-bcb9-a914fb5889e3", + "metadata": {}, + "outputs": [], + "source": [ + "#list = glob('/opt/dashboards/store/CCTP-SRSA-IP-20210831/*')\n", + "#for d in list : \n", + "# print(d)\n", + "# uuidStr = d.replace('/opt/dashboards/store/CCTP-SRSA-IP-20210831/','')\n", + "# createOnto(uuidStr)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47f67a4b-4d6c-4192-93b1-e697fd2c0e73", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18d75469-934f-4021-9126-ca0c1dcd4d98", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} -- GitLab