introduce unified ETL pipeline and API based data retrieval

.gitignore

@@ -1,4 +1,8 @@
 __pycache__/
 bibliovenv/
 Bibenv/
-.idea/
+.idea/
+venv/
+venv311/
+__pycache__/
+*.pyc

app.py

@@ -63,12 +63,13 @@
 from shiny import reactive, render
 from shinywidgets import render_widget
 from shiny.express import ui, input, render
+from www.services.standardizer import convert2df
+from www.services.api_retrievers import fetch_openalex, fetch_pubmed
 
 # Setup the Directory for static assets - optimized for performance
 base_dir = tempfile.gettempdir()  # Use system temp dir instead of creating new temp file
 express.app_opts(static_assets=base_dir, debug=False)
-
-# --- Toggle button ---
+sidebar_open = reactive.Value(False)# --- Toggle button ---
 # This button toggles the visibility of the sidebar(s) in the UI.
 ui.tags.button("☰", id="toggleSidebar", class_="sidebar-toggle")
 
@@ -751,15 +752,17 @@ def mostra():
                         reset_all_analyses()  # Reset analysis results when sample is loaded
 
                     @render.express()
-                    @reactive.event(input.Dataset)
                     def show_data():
                         text = get_data(input, database, df, reset_all_analyses)
                         text
-                    ui.HTML(init_itables())
-
+
                     @render.ui
-                    @reactive.event(input.start_button)
                     def show_table():
+                        data = df.get()
+
+                        if data is None:
+                            return ui.p("Please upload a dataset")
+
                         table_ui, _, _ = get_table(database, df)
                         return table_ui
 
@@ -853,9 +856,134 @@ def indicator_types_ui_all():
                     """
                 ),
 
-        with ui.nav_panel("None", value="API"):
-            ui.h3("🚧 Warning: API is under construction 🚧")
+        with ui.nav_panel("API", value="API"):
+
+            with ui.card(id="api_card", style="margin: 20px; padding: 20px; border-radius: 10px; box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1); background-color: #f9f9f9;"):
+                ui.tags.script("""
+                function hideApiCard() {
+                    const card = document.getElementById("api_card");
+                    if (card) card.style.display = "none";
+                }
+                """)
+
+                with ui.tags.div():  
+                    ui.h3("🌐 API Data Import", style="color: #5567BB;")
+                    ui.input_text("api_topic", "Search Topic")
+
+                    ui.input_select(
+                        "api_source",
+                        "Source",
+                        {
+                            "openalex": "OpenAlex",
+                            "pubmed": "PubMed"
+                        }
+                    )
+
+                    ui.input_numeric("api_limit", "Limit", value=50)
+                    ui.input_action_button("api_start", "Fetch Data", icon=ICONS["play"])
+
+
+            @render.express()
+            @reactive.event(input.api_start)
+            def api_load():
+
+                topic = input.api_topic()
+                source = input.api_source()
+                limit = input.api_limit()
+
+                if not topic:
+                    ui.notification_show("⚠️write a topic to search", duration=5, close_button=True)
+                    return
+
+                if source == "openalex":
+                    data = fetch_openalex(topic, limit)
+                    data = convert2df(
+                        source="openalex",
+                        raw_df=data
+                    )
+
+                elif source == "pubmed":
+                    data = fetch_pubmed(topic, limit)
+                    data = convert2df(
+                        source="pubmed_api",
+                        raw_df=data
+                    )
+
+                ui.markdown(f"<h3 style='text-align:center; color: #5567BB;'>Data of {source}</h3>")
+                df.set(data)
+                ui.tags.script("hideApiCard();")
+                database= f"{source} API"
+                reset_all_analyses()
+
+                @render.express()
+                def show_data():
+                    text = get_data(input, database, df, reset_all_analyses)
+                    text
 
+                @render.ui
+                def show_table():
+                    data = df.get()
+
+                    if data is None:
+                        return ui.p("Please upload a dataset")
+
+                    table_ui, _, _ = get_table(database, df)
+                    return table_ui
+
+                ui.notification_show(
+            f"✅ Data loaded from {source} (API)",
+            duration=5
+    )
+                # -------- ADVICE BUTTON --------
+                @render.ui
+                @reactive.event(input.advice_modal_completeness)
+                def show_advice_notification():
+                    return ui.notification_show(
+                        ui.div(
+                            ui.h4("Your metadata have no critical issues", style="font-size: 30px; text-align: center;"),
+                            ui.input_action_button("close_advice_modal_notification", "OK",
+                                                style="display: block; margin: 20px auto;")
+                        ),
+                        duration=None,  # La notifica rimane finché non viene chiusa
+                        close_button=False,  # Disabilita la X per la chiusura
+                        id="advice_modal_notification",
+                    )
+
+                # Aggiungi l'evento di chiusura al bottone OK
+                @reactive.effect
+                @reactive.event(input.close_advice_modal_notification)
+                def close_advice_notification():
+                    ui.notification_remove(id="advice_modal_notification")
+
+                # -------- REPORT BUTTON --------
+                @render.ui
+                @reactive.event(input.report_modal_completeness)
+                def show_missing_data_report():
+                    _, missingData, _ = get_table(database, df, modal=False)
+                    dataframe = pd.read_html(io.StringIO(missingData))
+                    report_excel.set(add_to_report(report_choices, report_excel, [dataframe[0]], [], "missingdata"))
+                    selection.set(selection.get() + (f"{list(report_choices.get().keys())[-1]}",))
+                    return ui.notification_show("✅ Missing data added to report", duration=5, close_button=False)
+
+                # -------- SAVE BUTTON --------
+                completeness_table_download_folder = str(Path.home() / "Downloads")
+                todaydate = datetime.today().strftime("%Y-%m-%d")
+                completeness_table_image_path = os.path.join(completeness_table_download_folder, f"missingDataTable-{todaydate}.png")
+                @render.ui
+                @reactive.event(input.save_modal_completeness)
+                def save_dataframe_image():
+                    _, _, fig = get_table(database, df, dpi=dpi.get(), modal=False)
+                    fig.write_image(completeness_table_image_path)
+                    return ui.notification_show(f"✅ Missing data image saved into {completeness_table_image_path}", duration=5, close_button=False)
+
+            # Loader indicator
+            @render.ui
+            def indicator_types_ui():
+                return ui.busy_indicators.use(
+                    spinners=input.api_start() > 0
+            )
+
+
         with ui.nav_panel("None", value="collections"):
             ui.h3("🚧 Warning: Merge Collection is under construction 🚧")
 
@@ -8185,8 +8313,10 @@ def update_plot_settings():
 
 # --- Sidebar Management ---
 @render.express()
-@reactive.event(input.start_button)
 def toggle_sidebar():
+    if df.get() is None:
+        return
+
     with ui.tags.div(id="sidebar_2", class_="custom-sidebar"):
         with ui.accordion(id="sidebar_accordion_data", multiple=False, open=False):
             # Info Section
@@ -8301,6 +8431,14 @@ def toggle_sidebar():
             """
             )
 
+    ui.tags.script("""
+        setTimeout(function() {
+            if (typeof setSidebarState === "function") {
+                setSidebarState(true);
+            }
+        }, 0);
+    """)
+
 
 # --- Javascript for Sidebar ---
 ui.tags.script("""
@@ -8309,9 +8447,18 @@ def toggle_sidebar():
         const sidebar = document.getElementById("sidebar");
         const sidebar_2 = document.getElementById("sidebar_2");
         const content = document.getElementById("mainContent");
-        if (sidebar) sidebar.classList.toggle("sidebar-hidden", !show);
-        if (sidebar_2) sidebar_2.classList.toggle("sidebar-hidden", !show);
-        if (content) content.classList.toggle("full-width", !show);
+
+        if (sidebar) {
+            sidebar.classList.toggle("sidebar-hidden", !show);
+        }
+
+        if (sidebar_2 && sidebar_2.classList) {
+            sidebar_2.classList.toggle("sidebar-hidden", !show);
+        }
+
+        if (content && content.classList) {
+            content.classList.toggle("full-width", !show);
+        }
     }
 
     // Hide sidebars on page load
@@ -8320,33 +8467,45 @@ def toggle_sidebar():
     });
 
     // Toggle both sidebars on button click
-    document.getElementById("toggleSidebar").addEventListener("click", function() {
-        const sidebar = document.getElementById("sidebar");
-        // If either sidebar is visible, hide both; otherwise, show both
-        const isVisible = sidebar && !sidebar.classList.contains("sidebar-hidden");
-        setSidebarState(!isVisible);
+    document.addEventListener("click", function(e) {
+        if (e.target && e.target.id === "toggleSidebar") {
+
+            const sidebar = document.getElementById("sidebar");
+            const isVisible = sidebar && !sidebar.classList.contains("sidebar-hidden");
+
+            setSidebarState(!isVisible);
+        }
     });
 
-    // Listen for Shiny events that might add/remove sidebar_2 dynamically
-    // and keep them in sync
+    // Safe MutationObserver (prevents null crash)
     const observer = new MutationObserver(function(mutations) {
         mutations.forEach(function(mutation) {
-            if (mutation.addedNodes.length > 0 || mutation.removedNodes.length > 0) {
-                // Always keep both sidebars in the same state
-                const sidebar = document.getElementById("sidebar");
-                const sidebar_2 = document.getElementById("sidebar_2");
-                if (sidebar && sidebar_2) {
-                    const sidebarHidden = sidebar.classList.contains("sidebar-hidden");
-                    sidebar_2.classList.toggle("sidebar-hidden", sidebarHidden);
-                }
+
+            const sidebar = document.getElementById("sidebar");
+            const sidebar_2 = document.getElementById("sidebar_2");
+
+            if (!sidebar || !sidebar_2) return;
+
+            if (sidebar.classList && sidebar_2.classList) {
+                const sidebarHidden = sidebar.classList.contains("sidebar-hidden");
+                sidebar_2.classList.toggle("sidebar-hidden", sidebarHidden);
             }
         });
     });
+
     observer.observe(document.body, { childList: true, subtree: true });
 
-    // Show both sidebars when 'start_button' is clicked
+    // Show sidebars when buttons clicked (safe)
     document.addEventListener("click", function(e) {
-        if (e.target && e.target.id === "start_button") {
+        const target = e.target && e.target.closest
+            ? e.target.closest("#start_button, #api_start")
+            : e.target;
+
+        if (
+            target &&
+            (target.id === "start_button" || target.id === "api_start")
+        ) {
+            console.log("Start button clicked - showing sidebars");
             setSidebarState(true);
         }
     });

functions/get_affiliationproductionovertime.py

@@ -12,7 +12,10 @@ def get_affiliation_production_over_time(df, top_k_affiliations):
     Returns:
         A Plotly figure object representing the affiliation's production over time.
     """
-    data = df.get()
+    if hasattr(df, "get") and not isinstance(df, pd.DataFrame):
+        data = df.get()
+    else:
+        data = df.copy()
 
     AFF = data["AU_UN"].dropna().apply(lambda x: [aff for aff in x if aff.strip() != ""])
     nAFF = [len(aff) for aff in AFF]

functions/get_annualproduction.py

@@ -11,7 +11,13 @@ def get_annual_production(df):
     Returns:
         A Plotly figure object representing the annual scientific production.
     """
-    data = df.get()
+    if hasattr(df, "get") and not isinstance(df, pd.DataFrame):
+        data = df.get()
+    else:
+        data = df.copy()
+
+    data["PY"] = pd.to_numeric(data["PY"], errors="coerce").fillna(0).astype(int)
+    data = data[data["PY"] > 0]
 
     # Calculate the number of publications per year
     publications_per_year = data["PY"].value_counts().sort_index().reset_index()

functions/get_authorlocalimpact.py

@@ -13,7 +13,10 @@ def get_authors_local_impact(df, num_of_authors_local_impact, author_local_impac
     Returns:
         A Plotly figure object and a DataFrame of the most impactful sources.
     """
-    df = df.get()
+    if hasattr(df, "get") and not isinstance(df, pd.DataFrame):
+        df = df.get()
+    else:
+        df = df.copy()
     today = pd.Timestamp.now().year
 
     # Ensure 'TC' and 'PY' are numeric

functions/get_authorproductionovertime.py

@@ -16,7 +16,10 @@ def get_author_production_over_time(df, top_k_authors):
         table_authors_production (pd.DataFrame): Table summarizing authors' production with TC and TCpY.
         table_documents (pd.DataFrame): Detailed table with additional document information.
     """
-    data = df.get()
+    if hasattr(df, "get") and not isinstance(df, pd.DataFrame):
+        data = df.get()
+    else:
+        data = df.copy()
 
     # Ensure "PY" is numeric
     data["PY"] = pd.to_numeric(data["PY"], errors="coerce")

functions/get_averagecitations.py

@@ -11,7 +11,10 @@ def get_average_citations(df):
     Returns:
         A Plotly figure object representing the average citations per year.
     """
-    data = df.get()
+    if hasattr(df, "get") and not isinstance(df, pd.DataFrame):
+        data = df.get()
+    else:
+        data = df.copy()
 
     # Calculate the current year
     current_year = pd.Timestamp.now().year + 1

functions/get_bradfordlaw.py

@@ -12,7 +12,10 @@ def get_bradford_law(df):
         A Plotly figure object and a DataFrame of the Bradford's Law zones.
     """
     # Sort data by frequency of occurrence (equivalent to R's sort(table(M$SO), decreasing = TRUE))
-    data = df.get()
+    if hasattr(df, "get") and not isinstance(df, pd.DataFrame):
+        data = df.get()
+    else:
+        data = df.copy()
     source_counts = data["SO"].value_counts()
 
     # Total number of sources

functions/get_correspondingauthorcountries.py

@@ -15,7 +15,10 @@ def get_corresponding_author_countries(df, top_k_countries):
     # Estrai i metadati "AU_CO" e "AU1_CO" e verifica il tipo di dati
     df = metaTagExtraction(df, Field="AU_CO")  # Assumendo che `metaTagExtraction` sia già definita
     df = metaTagExtraction(df, Field="AU1_CO")
-    data = df.get()  # Se `df` è un oggetto reattivo
+    if hasattr(df, "get") and not isinstance(df, pd.DataFrame):
+        data = df.get()
+    else:
+        data = df.copy()  # Se `df` è un oggetto reattivo
 
     # Assicurati che le colonne siano di tipo stringa e rimuovi righe con valori mancanti
     data = data.dropna(subset=["AU1_CO", "AU_CO"])

functions/get_countriesproductionovertime.py

@@ -13,7 +13,10 @@ def get_countries_production_over_time(df, top_k_countries):
         A Plotly figure object representing the country's production over time.
     """
     df = metaTagExtraction(df, "AU_CO")
-    data = df.get()
+    if hasattr(df, "get") and not isinstance(df, pd.DataFrame):
+        data = df.get()
+    else:
+        data = df.copy()
 
     AFF = pd.Series(data["AU_CO"]).dropna().apply(lambda x: [aff.strip() for aff in x if aff.strip() != ""])
     nAFF = [len(aff) for aff in AFF]