From 08643148edd086a683324c1e126e17054fbc8ddb Mon Sep 17 00:00:00 2001 From: Fernando Aguilar Date: Mon, 1 Dec 2025 16:42:37 +0100 Subject: [PATCH 1/4] Handle API timeouts and render data tests --- fair_eva_web_client/app.py | 68 ++++++++++++++++++++-- fair_eva_web_client/templates/eval.html | 12 ++++ fair_eva_web_client/templates/timeout.html | 42 +++++++++++++ 3 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 fair_eva_web_client/templates/timeout.html diff --git a/fair_eva_web_client/app.py b/fair_eva_web_client/app.py index 9edbbdd..8b0ec55 100644 --- a/fair_eva_web_client/app.py +++ b/fair_eva_web_client/app.py @@ -32,8 +32,8 @@ from flask_babel import lazy_gettext as _l from flask_wtf import FlaskForm -from wtforms import StringField, SelectField, SubmitField -from wtforms.validators import DataRequired +from wtforms import HiddenField, StringField, SelectField, SubmitField, TextAreaField +from wtforms.validators import DataRequired, Email @@ -284,6 +284,16 @@ class IdentifierForm(FlaskForm): plugin = SelectField("Select plugin", choices=[], validators=[DataRequired()]) submit = SubmitField("Evaluate") + +class EmailRequestForm(FlaskForm): + """Form shown when the API takes too long and the user wants an email copy.""" + + item_id = HiddenField(validators=[DataRequired()]) + plugin = HiddenField(validators=[DataRequired()]) + email = StringField(_l("Email"), validators=[DataRequired(), Email()]) + notes = TextAreaField(_l("Additional notes")) + submit = SubmitField(_l("Send results by email")) + ############################################################################### # Application factory and routes ############################################################################### @@ -487,19 +497,41 @@ def evaluator(): # Ajusta el endpoint real si es distinto: endpoint = f"{base}/v1.0/rda/rda_all" payload: Dict[str, Any] = {"id": item_id, "repo": repo, "lang": g.language} - resp = requests.post(endpoint, json=payload, timeout=30) + # La API puede tardar; damos hasta 2 minutos antes de ofrecer el plan B + resp = requests.post(endpoint, json=payload, timeout=120) resp.raise_for_status() resp_json = resp.json() for k, v in resp_json.items(): if k != "evaluator_logs": result_data = v break + except requests.Timeout: + email_form = EmailRequestForm() + email_form.item_id.data = item_id + email_form.plugin.data = plugin + return render_template( + "timeout.html", + form=email_form, + item_id=item_id, + plugin=plugin, + ) except Exception as exc: return render_template("error.html", error=f"Error loading evaluation data: {exc}") if not result_data: return render_template("error.html", error="No evaluation data returned.") + data_test_html: Optional[str] = None + data_tests = result_data.get("data_test") if isinstance(result_data, dict) else None + if isinstance(data_tests, dict): + for _, block in data_tests.items(): + if not isinstance(block, dict): + continue + msg = block.get("msg") + if isinstance(msg, str) and msg.strip(): + data_test_html = msg + break + # -------------------------------- # 2) Puntuaciones (tu función) # -------------------------------- @@ -651,7 +683,7 @@ def _tests_list(group: Dict[str, Any]) -> List[Dict[str, Any]]: div="", script_f="", div_f="", - data_test=None, + data_test=data_test_html, # --- moderno --- resource_id=resource_id, plugin_name=plugin_name, @@ -669,6 +701,34 @@ def not_found(): return render_template("not-found.html") + @app.route("/request-email", methods=["POST"], endpoint="request_email") + def request_email(): + form = EmailRequestForm() + if form.validate_on_submit(): + log_dir = os.path.join(os.path.dirname(__file__), "data") + os.makedirs(log_dir, exist_ok=True) + log_path = os.path.join(log_dir, "email_requests.log") + with open(log_path, "a", encoding="utf-8") as fh: + fh.write( + f"{datetime.utcnow().isoformat()}Z\t{form.item_id.data}\t{form.plugin.data}\t{form.email.data}\t{(form.notes.data or '').replace(chr(10), ' ')}\n" + ) + return render_template( + "timeout.html", + form=form, + item_id=form.item_id.data, + plugin=form.plugin.data, + submitted=True, + ) + + return render_template( + "timeout.html", + form=form, + item_id=form.item_id.data, + plugin=form.plugin.data, + submitted=False, + ) + + @app.route("/es/faq", endpoint="faq_es") @app.route("/en/faq", endpoint="faq_en") def faq(): diff --git a/fair_eva_web_client/templates/eval.html b/fair_eva_web_client/templates/eval.html index 2a784cc..15b7c16 100644 --- a/fair_eva_web_client/templates/eval.html +++ b/fair_eva_web_client/templates/eval.html @@ -484,6 +484,18 @@

{% endfor %} + + + {% if data_test %} +
+

{{ _('Additional data tests') }}

+
+
+ {{ data_test | safe }} +
+
+
+ {% endif %} {% endblock %} diff --git a/fair_eva_web_client/templates/timeout.html b/fair_eva_web_client/templates/timeout.html new file mode 100644 index 0000000..ef5c319 --- /dev/null +++ b/fair_eva_web_client/templates/timeout.html @@ -0,0 +1,42 @@ +{% extends 'base_modern.html' %} + +{% block body %} +
+
+
+

{{ _('The evaluation is taking longer than expected') }}

+

{{ _('You can wait a bit more or leave us an email to send you the results as soon as they are ready.') }}

+ + {% if submitted %} + + {% endif %} + +
+ {{ form.hidden_tag() }} + {{ form.item_id() }} + {{ form.plugin() }} +
+ + {{ form.email(class_='form-control', placeholder='you@example.com') }} + {% if form.email.errors %} +
{{ form.email.errors[0] }}
+ {% endif %} +
+
+ + {{ form.notes(class_='form-control', rows=3) }} +
+ + {{ _('Back to start') }} +
+ +
+
{{ _('Identifier') }}: {{ item_id }}
+
{{ _('Plugin') }}: {{ plugin }}
+
+
+
+
+{% endblock %} From 6bef9c463817a66fbaeafb8ffcb0bfb59aeee815 Mon Sep 17 00:00:00 2001 From: Fernando Aguilar Date: Mon, 1 Dec 2025 18:27:04 +0100 Subject: [PATCH 2/4] Switch default plugin to GBIF and add email validator --- fair_eva_web_client/app.py | 2 +- pyproject.toml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fair_eva_web_client/app.py b/fair_eva_web_client/app.py index 8b0ec55..05a8792 100644 --- a/fair_eva_web_client/app.py +++ b/fair_eva_web_client/app.py @@ -416,7 +416,7 @@ def catch_all(path): AVAILABLE_PLUGINS = load_available_plugins() if not AVAILABLE_PLUGINS: AVAILABLE_PLUGINS = [ - ("signposting", "Signposting (Zenodo/CSIC)"), + ("gbif", "GBIF"), ("oai_pmh", "OAI-PMH"), ("ai4os", "AI4EOSC Plugin"), ] diff --git a/pyproject.toml b/pyproject.toml index 4de4459..dfd0a14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,8 @@ dependencies = [ "requests>=2.0", "Babel", "flask-babel", - "flask-wtf" + "flask-wtf", + "email-validator" ] [project.scripts] From 729b6eb9cae9980d6050e34b6967709c830dcacc Mon Sep 17 00:00:00 2001 From: Fernando Aguilar Date: Mon, 1 Dec 2025 18:43:36 +0100 Subject: [PATCH 3/4] Queue background evaluator call for email requests --- fair_eva_web_client/app.py | 62 +++++++++++++++++++++- fair_eva_web_client/templates/timeout.html | 4 +- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/fair_eva_web_client/app.py b/fair_eva_web_client/app.py index 05a8792..3f98da0 100644 --- a/fair_eva_web_client/app.py +++ b/fair_eva_web_client/app.py @@ -10,6 +10,7 @@ from __future__ import annotations from datetime import datetime +import threading import importlib import json import os @@ -705,12 +706,71 @@ def not_found(): def request_email(): form = EmailRequestForm() if form.validate_on_submit(): + lang = session.get("lang") or getattr(g, "language", "en") + + def queue_background_request( + cfg: Settings, item_id: str, plugin: str, email: str, notes: str, lang: str + ) -> None: + """Call the evaluator API in the background and persist the result.""" + + base = cfg.api_url.rstrip("/") + f":{cfg.api_port}" + endpoint = f"{base}/v1.0/rda/rda_all" + payload: Dict[str, Any] = { + "id": item_id, + "repo": plugin, + "lang": lang, + } + log_dir = os.path.join(os.path.dirname(__file__), "data") + os.makedirs(log_dir, exist_ok=True) + log_path = os.path.join(log_dir, "email_requests.log") + try: + resp = requests.post(endpoint, json=payload, timeout=600) + resp.raise_for_status() + resp_json = resp.json() + result_path = os.path.join( + log_dir, + f"email_result_{datetime.utcnow().isoformat().replace(':', '-')}.json", + ) + with open(result_path, "w", encoding="utf-8") as out: + json.dump( + { + "requested_at": datetime.utcnow().isoformat() + "Z", + "item_id": item_id, + "plugin": plugin, + "email": email, + "notes": notes, + "response": resp_json, + }, + out, + ensure_ascii=False, + indent=2, + ) + status = "queued" + except Exception as exc: + status = f"error: {exc}" + with open(log_path, "a", encoding="utf-8") as fh: + fh.write( + f"{datetime.utcnow().isoformat()}Z\t{item_id}\t{plugin}\t{email}\t{notes.replace(chr(10), ' ')}\t{status}\n" + ) + + threading.Thread( + target=queue_background_request, + args=( + cfg, + form.item_id.data, + form.plugin.data, + form.email.data, + form.notes.data or "", + lang, + ), + daemon=True, + ).start() log_dir = os.path.join(os.path.dirname(__file__), "data") os.makedirs(log_dir, exist_ok=True) log_path = os.path.join(log_dir, "email_requests.log") with open(log_path, "a", encoding="utf-8") as fh: fh.write( - f"{datetime.utcnow().isoformat()}Z\t{form.item_id.data}\t{form.plugin.data}\t{form.email.data}\t{(form.notes.data or '').replace(chr(10), ' ')}\n" + f"{datetime.utcnow().isoformat()}Z\t{form.item_id.data}\t{form.plugin.data}\t{form.email.data}\t{(form.notes.data or '').replace(chr(10), ' ')}\tqueued\n" ) return render_template( "timeout.html", diff --git a/fair_eva_web_client/templates/timeout.html b/fair_eva_web_client/templates/timeout.html index ef5c319..b2c536b 100644 --- a/fair_eva_web_client/templates/timeout.html +++ b/fair_eva_web_client/templates/timeout.html @@ -7,9 +7,11 @@

{{ _('The evaluation is taking longer than expected') }}

{{ _('You can wait a bit more or leave us an email to send you the results as soon as they are ready.') }}

+

{{ _('If you submit the form, we will keep querying the evaluator in the background and email you once a response arrives, even if you close this page.') }}

+ {% if submitted %} {% endif %} From fb43c328d19017ad9f3f6cd8b0adb9fcef28091e Mon Sep 17 00:00:00 2001 From: Fernando Aguilar Date: Mon, 1 Dec 2025 18:52:02 +0100 Subject: [PATCH 4/4] Poll evaluator until response and email results --- fair_eva_web_client/app.py | 88 ++++++++++++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 9 deletions(-) diff --git a/fair_eva_web_client/app.py b/fair_eva_web_client/app.py index 3f98da0..652e848 100644 --- a/fair_eva_web_client/app.py +++ b/fair_eva_web_client/app.py @@ -9,12 +9,15 @@ """ from __future__ import annotations -from datetime import datetime +from datetime import datetime, timedelta +from email.message import EmailMessage import threading import importlib import json import os import sys +import smtplib +import time from dataclasses import dataclass from importlib import metadata from typing import Any, Dict, Optional, Tuple, List @@ -295,6 +298,44 @@ class EmailRequestForm(FlaskForm): notes = TextAreaField(_l("Additional notes")) submit = SubmitField(_l("Send results by email")) + +def send_email_message(to: str, subject: str, body: str) -> Tuple[bool, str]: + """Send a plaintext email using SMTP settings from the environment. + + Environment variables: + FAIR_EVA_SMTP_HOST: SMTP server hostname. + FAIR_EVA_SMTP_PORT: SMTP server port (default 587). + FAIR_EVA_SMTP_USER: Optional username for authentication. + FAIR_EVA_SMTP_PASSWORD: Optional password for authentication. + FAIR_EVA_SMTP_TLS: Enable STARTTLS when set to "1" (default). + """ + + host = os.getenv("FAIR_EVA_SMTP_HOST") + if not host: + return False, "FAIR_EVA_SMTP_HOST not configured" + + port = int(os.getenv("FAIR_EVA_SMTP_PORT", "587")) + username = os.getenv("FAIR_EVA_SMTP_USER") + password = os.getenv("FAIR_EVA_SMTP_PASSWORD") + use_tls = os.getenv("FAIR_EVA_SMTP_TLS", "1") != "0" + + message = EmailMessage() + message["To"] = to + message["Subject"] = subject + message["From"] = username or f"fair-eva@{host}" + message.set_content(body) + + try: + with smtplib.SMTP(host, port, timeout=30) as smtp: + if use_tls: + smtp.starttls() + if username and password: + smtp.login(username, password) + smtp.send_message(message) + return True, "sent" + except Exception as exc: # pragma: no cover - network errors + return False, str(exc) + ############################################################################### # Application factory and routes ############################################################################### @@ -711,7 +752,7 @@ def request_email(): def queue_background_request( cfg: Settings, item_id: str, plugin: str, email: str, notes: str, lang: str ) -> None: - """Call the evaluator API in the background and persist the result.""" + """Poll the evaluator API until it responds and send the result via email.""" base = cfg.api_url.rstrip("/") + f":{cfg.api_port}" endpoint = f"{base}/v1.0/rda/rda_all" @@ -720,13 +761,32 @@ def queue_background_request( "repo": plugin, "lang": lang, } + + max_wait_minutes = int(os.getenv("FAIR_EVA_EMAIL_WAIT_MINUTES", "10")) + poll_interval = int(os.getenv("FAIR_EVA_EMAIL_POLL_SECONDS", "15")) + deadline = datetime.utcnow() + timedelta(minutes=max_wait_minutes) + log_dir = os.path.join(os.path.dirname(__file__), "data") os.makedirs(log_dir, exist_ok=True) log_path = os.path.join(log_dir, "email_requests.log") - try: - resp = requests.post(endpoint, json=payload, timeout=600) - resp.raise_for_status() - resp_json = resp.json() + + resp_json: Optional[Dict[str, Any]] = None + last_error: Optional[str] = None + + while datetime.utcnow() < deadline: + try: + resp = requests.post(endpoint, json=payload, timeout=60) + resp.raise_for_status() + resp_json = resp.json() + break + except Exception as exc: + last_error = str(exc) + time.sleep(poll_interval) + + status: str + result_path: Optional[str] = None + + if resp_json is not None: result_path = os.path.join( log_dir, f"email_result_{datetime.utcnow().isoformat().replace(':', '-')}.json", @@ -745,9 +805,19 @@ def queue_background_request( ensure_ascii=False, indent=2, ) - status = "queued" - except Exception as exc: - status = f"error: {exc}" + subject = f"FAIR EVA results for {item_id}" + body = ( + "Your evaluation has finished.\n\n" + f"Identifier: {item_id}\n" + f"Plugin: {plugin}\n" + f"Notes: {notes}\n\n" + f"Full response:\n{json.dumps(resp_json, ensure_ascii=False, indent=2)}" + ) + sent, detail = send_email_message(email, subject, body) + status = "email_sent" if sent else f"email_failed: {detail}" + else: + status = f"error: {last_error or 'timeout waiting for evaluator'}" + with open(log_path, "a", encoding="utf-8") as fh: fh.write( f"{datetime.utcnow().isoformat()}Z\t{item_id}\t{plugin}\t{email}\t{notes.replace(chr(10), ' ')}\t{status}\n"