Add batch_unlizeToNotebook / add ipynb data / commented code to execute...

Add batch_unlizeToNotebook / add ipynb data / commented code to execute notebooks in conversion / rescale svg in notebooks

Add batch_unlizeToNotebook / add ipynb data / commented code to execute...
1042e1cc · David Rouquet · a3d52116 · 1042e1cc · 1042e1cc · 1042e1cc
Commit 1042e1cc authored 5 years ago by David Rouquet
--- a/data/unl.ru/ipynb.zip
+++ b/data/unl.ru/ipynb.zip
--- a/scripts/batch_unlizeToNotebook.sh
+++ b/scripts/batch_unlizeToNotebook.sh
+#!/bin/bash
+for filepath in unl/*; do
+	echo "$filepath"
+	python unlizeToNotebook.py "$filepath" "$(basename "$filepath" .xml).ipynb"
+    done
--- a/scripts/unlizeToNotebook.py
+++ b/scripts/unlizeToNotebook.py
@@ -2,7 +2,7 @@
 import nbformat as nbf
 from nbformat.v4 import new_code_cell
-import nbconvert as nbc
+#from nbconvert.preprocessors import ExecutePreprocessor
 import click
 from lxml import etree, objectify
 from unlizeXml import remove_namespace, unlize, nestedBody2Str
@@ -32,6 +32,10 @@ def unlizeXmlNb(input, output, template, lang, dry_run):
                addCell(nb, unl)
            else:
                addCell(nb, getText(node, 'unl'))
+    # We execute the notebook
+    #resources = {}
+    #exec_nb, resources = ExecutePreprocessor(timeout=-1).preprocess(nb, resources)
+    # Save
    with open(output, 'w') as f:
        nbf.write(nb, f)

--- a/scripts/unlizeXmlNbSample.ipynb
+++ b/scripts/unlizeXmlNbSample.ipynb
@@ -11,12 +11,12 @@
    "import re\n",
    "import requests\n",
    "from subprocess import Popen, PIPE, STDOUT\n",
-    "from IPython.core.display import SVG"
+    "from IPython.display import SVG, display, HTML"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -73,6 +73,12 @@
    "    svg = regex.search(html).group()\n",
    "    return(svg)\n",
    "\n",
+    "def svg_to_fixed_width_html_image(svg, width=\"100%\"):\n",
+    "    html_template='<img width=\"{}\">{}</img>'\n",
+    "    text = html_template.format(width, svg)\n",
+    "    return HTML(text)\n",
+    "\n",
+    "\n",
    "def displayUnl(unldata) :\n",
    "# We generate protoSVG because whent there are several sentences, \n",
    "# a string composed of several concatenated SVG is produced (not a valid SVG).\n",
@@ -88,7 +94,7 @@
    "        try:\n",
    "            #svg = unl2dotWeb(unl)\n",
    "            svg = unl2dot(unl, \"unl2rdf-app-1.0-SNAPSHOT-jar-with-dependencies.jar\")\n",
-    "            display(SVG(svg))\n",
+    "            display(svg_to_fixed_width_html_image(svg))\n",
    "        except Exception as e :\n",
    "            print(e)"
   ]

 %% Cell type:code id: tags:
 ``` python
 import tempfile
 import os
 import re
 import requests
 from subprocess import Popen, PIPE, STDOUT
-from IPython.core.display import SVG
+from IPython.display import SVG, display, HTML
 ```
 %% Cell type:code id: tags:
 ``` python
 def unl2dot(text, path):
    with tempfile.NamedTemporaryFile() as temp:
        out_name = os.path.basename(temp.name)
        out_dir = os.path.dirname(temp.name)
    with tempfile.NamedTemporaryFile(mode="w") as in_file:
        # Remove CRLF and flush output to avoid java errors
        in_file.write(text.replace("\r\n", "\n"))
        in_file.flush()
        # Run java parser
        cmd = ['java', '-jar', path,
               '--input-file', in_file.name,
               '--output-Dir', out_dir, '--output-file', out_name,
               '--output-type', 'dot']
        with Popen(cmd, stdout=PIPE, stderr=STDOUT) as p:
            p.wait()
            p.stdout.flush()
            if p.returncode != 0:
                print("Error in unl2rdf: \n\n"+p.stdout.read().decode())
                print('UNL;')
                print(text)
    # generate dot output
    fname = '{}/{}.dot'.format(out_dir, out_name)
    cmd = ['dot', '-Tsvg', fname]
    with Popen(cmd, stdout=PIPE, stderr=PIPE) as p:
        p.wait()
        if p.returncode != 0:
            print("Error creating svg: \n\n"+p.stderr.read().decode())
            print('UNL:')
            print(text)
            try:
                with open(fname) as f:
                    print('DOT:')
                    print(f.read())
            except FileNotFoundError:
                pass
        else:
            svg = p.stdout.read().decode()
            os.remove(fname)
            return svg
    return ""
 def unl2dotWeb(unldata) :
    data={'unl': unldata, 'outputs':['dot', 'svg']}
    try:
        r = requests.post('https://unl.demo.tetras-libre.fr/unl2rdf', data=data)
    except Exception as e:
        return 'Error calling https://unl.demo.tetras-libre.fr/unl2rdf : "{error}"'.format(error=e)
    html=r.text
    # On utilise une regex au lieu de parser le html car ce dernier est mal formé
    regex = re.compile('<svg.*svg>',re.MULTILINE|re.DOTALL)
    svg = regex.search(html).group()
    return(svg)
+def svg_to_fixed_width_html_image(svg, width="100%"):
+    html_template='<img width="{}">{}</img>'
+    text = html_template.format(width, svg)
+    return HTML(text)
 def displayUnl(unldata) :
 # We generate protoSVG because whent there are several sentences,
 # a string composed of several concatenated SVG is produced (not a valid SVG).
 # We must then split the string to obtain several valid SVG to display.
    sep = "[/S]\n"
    unldataArray = [x+sep for x in unldata.split(sep)]
    unldataArray.pop()
    for unl in unldataArray :
        regex = re.compile('\{org:..\}\n(.*)\n{\/org\}',re.MULTILINE|re.DOTALL)
        text = regex.search(unl).group(1)
        print("\n"+text+"\n")
        # Keep one of the two lines below depending if you want to use a local jar or a webservice for unltools
        try:
            #svg = unl2dotWeb(unl)
            svg = unl2dot(unl, "unl2rdf-app-1.0-SNAPSHOT-jar-with-dependencies.jar")
-            display(SVG(svg))
+            display(svg_to_fixed_width_html_image(svg))
        except Exception as e :
            print(e)
 ```