[biblio-ref] add new route (pps-search)

.gitignore

@@ -145,3 +145,5 @@ __pycache__/
 # Local files
 services/data-termsuite/NOTES.md
 services/loterre-resolvers/D63.hurl
+services/biblio-ref/v1/pps.csv
+

services/biblio-ref/Dockerfile

@@ -16,8 +16,9 @@ COPY --chown=daemon:daemon . /app/public
 COPY --chown=daemon:daemon ./config.json /app/config.json
 
 # Download the list of annulled papers from PPS + process it
-RUN curl -go /tmp/pps.csv 'https://dbrech.irit.fr/pls/apex/f?p=9999:300::IR[allproblematicpapers]_CSV'
-RUN awk -F '","' 'index($1,"annulled") {print $2}' /tmp/pps.csv > /app/public/v1/annulled.csv && \
-    rm /tmp/pps.csv && \
-    python v1/csv2pickle.py v1/annulled.csv && \
-    rm /app/public/v1/annulled.csv
+RUN curl -go /tmp/pps.csv 'https://dbrech.irit.fr/pls/apex/f?p=9999:300::IR[allproblematicpapers]_CSV' && \
+    awk -F '","' 'index($1,"annulled") {print $2}' /tmp/pps.csv > /app/public/v1/annulled.csv && \
+    python v1/csv2pickle.py /app/public/v1/annulled.csv && \
+    rm /app/public/v1/annulled.csv && \
+    python v1/csv2pickle-all.py /tmp/pps.csv && \
+    rm /tmp/pps.csv

services/biblio-ref/README.md

@@ -13,12 +13,21 @@ Si vous ne disposez pas de token, vous pouvez supprimer les  headers des requêt
 
 Pour les tests en local, il peut être contraignant de télécharger le contenu du PPS à chaque build. Nous pouvons donc supprimer temporairement le [téléchargement du dockerfile](https://github.com/Inist-CNRS/web-services/blob/b68b9b50f9fe17caeb8182c86b15fa624108397d/services/biblio-ref/Dockerfile#L20) et exploiter le résultat directement en local. Pour l'obtenir une fois "pour toute" en local, voici la commande (cas où le dépôt git `web-services` est clôné dans `~/workspace`):
 
+On le téléchage une seule fois :
+
+```sh
+curl -go ~/workspace/web-services/services/biblio-ref/v1/pps.csv 'https://dbrech.irit.fr/pls/apex/f?p=9999:300::IR[allproblematicpapers]_CSV' 
+```
+
+Puis on modifie la ligne là du dockerfile :
+
 ```sh
-curl -go /tmp/pps.csv 'https://dbrech.irit.fr/pls/apex/f?p=9999:300::IR[allproblematicpapers]_CSV' && \
-awk -F '","' 'index($1,"annulled") {print $2}' /tmp/pps.csv > ~/workspace/web-services/services/biblio-ref/v1/annulled.csv && \
-rm /tmp/pps.csv && \
-python ~/workspace/web-services/services/biblio-ref/v1/csv2pickle.py ~/workspace/web-services/services/biblio-ref/v1/annulled.csv && \
-rm ~/workspace/web-services/services/biblio-ref/v1/annulled.csv
+COPY v1/pps.csv /tmp/pps.csv
+RUN awk -F '","' 'index($1,"annulled") {print $2}' /tmp/pps.csv > /app/public/v1/annulled.csv && \
+    python v1/csv2pickle.py /app/public/v1/annulled.csv && \
+    rm /app/public/v1/annulled.csv && \
+    python v1/csv2pickle-all.py /tmp/pps.csv && \
+    rm /tmp/pps.csv
 ```
 
 Ne pas push ce fichier en l'état sur GIT, le fichier peut ainsi se mettre à jour à chaque version.

services/biblio-ref/examples.http

@@ -94,3 +94,28 @@ Content-Type: application/json
     "value": {"reference": "L.K. Bieniasz, Theory and highly accurate computation of non- limiting chronoamperometric currents for the ErevCirr reaction mechanism at cylindrical wire electrodes. J. Electroanal. Chem. 895, 115410 (2021)"}
   }
 ]
+
+###
+# @name v1ppsSearch
+POST {{host}}/v1/pps-search?indent=true
+content-type: application/json
+[
+  {
+    "value": "10.11591/ijeecs.v23.i3.pp1748-1760"
+  },
+  {
+    "value": "10.1111/coin.12508"
+  },
+  {
+    "value": "10.17762/msea.v70i2.13"
+  },
+  {
+    "value": "10.1189/87641.92158"
+  },
+  {
+    "value": [
+      "10.1056/nejm200106073442303",
+      "10.1016/b978-0-323-90638-8.00002-3"
+    ]
+  }
+]

services/biblio-ref/tests.hurl

@@ -280,3 +280,71 @@ jsonpath "$" count == 2
 jsonpath "$[*].value.from_crossref" exists
 jsonpath "$[*].value.raw_ref" exists
 jsonpath "$[*].value.matches_scores" exists
+
+
+# PSS route
+POST {{host}}/v1/pps-search?indent=true
+content-type: application/json
+[
+  {
+    "value": "10.11591/ijeecs.v23.i3.pp1748-1760"
+  },
+  {
+    "value": "10.1111/coin.12508"
+  },
+  {
+    "value": "10.17762/msea.v70i2.13"
+  },
+  {
+    "value": "10.1189/87641.92158"
+  },
+  {
+    "value": [
+      "10.1056/nejm200106073442303",
+      "10.1016/b978-0-323-90638-8.00002-3"
+    ]
+  }
+]
+
+
+HTTP 200
+[{
+    "value": {
+        "classes": [
+            "suspect",
+            "tortured"
+        ]
+    }
+},
+{
+    "value": {
+        "classes": [
+            "annulled",
+            "clayfeet",
+            "suspect",
+            "tortured"
+        ]
+    }
+},
+{
+    "value": {
+        "classes": [
+            "mathgen"
+        ]
+    }
+},
+{
+    "value": {
+        "classes": []
+    }
+},
+{
+    "value": {
+        "classes": [
+            [
+                "expression-of-concern"
+            ],
+            []
+        ]
+    }
+}]

services/biblio-ref/v1/bibref/bibref_functions.py

@@ -635,7 +635,7 @@ def biblio_ref(ref_biblio, retracted_doi=retracted_doi):
             reference_found = others_biblio_info["raw_ref"]
 
             # # # can be hallucinated
-            if len(doi)*1.5 < len(ref_biblio): 
+            if len(doi)*2 < len(ref_biblio): 
                 match_items_score, title_score, doi, potential_different_content = compare_pubinfo_refbiblio(others_biblio_info, ref_biblio)
                 if match_items_score < 2:
                     # We return "REFERENCE ASSOCIATED WITH THE DOI FROM CROSSREF >" when we suspect an hallucination
@@ -659,7 +659,7 @@ def biblio_ref(ref_biblio, retracted_doi=retracted_doi):
 
 
                 # # # can be hallucinated
-                if len(doi)*1.5 < len(ref_biblio): 
+                if len(doi)*2 < len(ref_biblio): 
                     match_items_score, title_score, doi, potential_different_content = compare_pubinfo_refbiblio(others_biblio_info, ref_biblio)
 
                     if match_items_score < 3:

services/biblio-ref/v1/csv2pickle-all.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+import csv
+import pickle
+import sys
+import json
+
+input_file = sys.argv[1]
+output_file = "/app/public/v1/all-pps-classes.pickle"
+
+detectors_dict = {}
+
+with open(input_file, 'r') as file:
+    reader = csv.DictReader(file)
+    for row in reader:
+        classes_str = row["Detectors"].strip().strip('"')
+        classes = [c.strip().lower() for c in classes_str.split(',')]
+        doi = row["Doi"].strip().strip('"').lower()
+        for classe in classes:
+            if doi not in detectors_dict:
+                detectors_dict[doi] = []
+            detectors_dict[doi].append(classe)
+
+with open(output_file, 'wb') as file:
+    pickle.dump(detectors_dict, file)
+
+
+with open("v1/testt.json", 'w') as file:
+    json.dump(detectors_dict, file)

services/biblio-ref/v1/csv2pickle.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import csv
 import pickle
 import sys

services/biblio-ref/v1/pps-search.ini

@@ -0,0 +1,53 @@
+# OpenAPI Documentation - JSON format (dot notation)
+mimeType = application/json
+
+post.operationId = post-v1-pps-search
+post.summary = Vérifie si un DOI est présent dans le PPS
+post.description = Vérifier si un ou plusieurs DOI est présent dans le PPS et retourne sa ou ses [Detectors](https://dbrech.irit.fr/pls/apex/f?p=9999:3::::::)
+post.tags.0 = biblio-ref
+post.requestBody.content.application/json.schema.$ref = #/components/schemas/JSONStream
+post.requestBody.required = true
+post.responses.default.content.application/json.schema.$ref =  #/components/schemas/JSONStream
+post.responses.default.description = Le champ `value` contient un booléen ou une liste de booléens indiquant pour chaque DOI sa/ses `detectors` dans le PPS.
+post.parameters.0.description = Indenter le JSON résultant
+post.parameters.0.in = query
+post.parameters.0.name = indent
+post.parameters.0.schema.type = boolean
+
+
+# Exemples
+post.requestBody.content.application/json.example.0.value = 10.11591/ijeecs.v23.i3.pp1748-1760
+post.requestBody.content.application/json.example.1.value = 10.1111/coin.12508
+post.requestBody.content.application/json.example.2.value = 10.17762/msea.v70i2.13
+post.requestBody.content.application/json.example.3.value = 10.1189/87641.92158
+post.requestBody.content.application/json.example.4.value.0 = 10.1056/nejm200106073442303
+post.requestBody.content.application/json.example.4.value.1 = 10.1016/b978-0-323-90638-8.00002-3
+post.responses.default.content.application/json.example.0.value.classes.0 = suspect
+post.responses.default.content.application/json.example.0.value.classes.1 = tortured
+post.responses.default.content.application/json.example.1.value.classes.0 = clayfeet
+post.responses.default.content.application/json.example.1.value.classes.1 = annulled
+post.responses.default.content.application/json.example.1.value.classes.2 = suspect
+post.responses.default.content.application/json.example.1.value.classes.3 = tortured
+post.responses.default.content.application/json.example.2.value.classes.0 = mathgen
+post.responses.default.content.application/json.example.3.value.classes = []
+post.responses.default.content.application/json.example.4.value.classes.0.0 = expression-of-concern
+post.responses.default.content.application/json.example.4.value.classes.1 = []
+
+
+[use]
+plugin = @ezs/spawn
+plugin = @ezs/basics
+
+[JSONParse]
+separator = *
+
+[expand]
+path = value
+size = 10
+
+[expand/exec]
+# command should be executable !
+command = ./v1/pps-search.py
+
+[dump]
+indent = env('indent', false)

services/biblio-ref/v1/pps-search.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+import json
+import sys
+import pickle
+
+with open("v1/all-pps-classes.pickle", "rb") as file:
+    all_classes = pickle.load(file)
+
+
+def get_classes_for_doi(doi):
+    return all_classes.get(doi, [])
+
+
+for line in sys.stdin:
+    data = json.loads(line)
+    doi = data["value"]
+
+    if isinstance(doi, str):
+        classes = get_classes_for_doi(doi)
+        if "id" in data:
+            output = {"id": data["id"], "value": {"classes": classes}}
+        else:
+            output = {"value": {"classes": classes}}
+
+    elif isinstance(doi, list):
+        classes_list = [get_classes_for_doi(elt) for elt in doi]
+        if "id" in data:
+            output = {"id": data["id"], "value": {"classes": classes_list}}
+        else:
+            output = {"value": {"classes": classes_list}}
+
+    else:
+        if "id" in data:
+            output = {"id": data["id"], "value": {"classes": []}}
+        else:
+            output = {"value": {"classes": []}}
+
+    json.dump(output, sys.stdout)
+    sys.stdout.write("\n")