feat: implement export_single_faq_page_to_md utility; enhance logging in ChatApp; add new log tags for NOT-ANSWERED and STREAMLIT

Author: yesidc

Dockerfile

@@ -7,6 +7,7 @@ WORKDIR /app
 RUN apt-get update && apt-get install -y \
     sudo \
     wget \
+    vim \
     gettext \
     && rm -rf /var/lib/apt/lists/*
 

data_ingestion/crud_ragflow.py

@@ -46,7 +46,7 @@ def insert_metadata_ragflow(
     base_url=None,
     api_key=None,
 ):
-    # TODO: Onls works with md files for now. Extend to other file types.
+    # TODO: Only works with md files for now. Extend to other file types.
     base_url = base_url or settings.vector_db_settings.settings.base_url
     api_key = api_key or os.getenv("RAGFLOW_API_KEY")
 

data_ingestion/detect_faq.py

@@ -313,6 +313,18 @@ async def run(
             logger.info(f"Failed URLs: {failed_urls}")
 
 
+async def export_single_faq_page_to_md(url: str) -> None:
+    """Utility function to export a single FAQ page to Markdown."""
+    config = FAQDetectorConfig()
+    detector = FAQDetector(config)
+    content = await detector.crawl_url(url)
+    if content and content.markdown:
+        detector.save_faq_content(content, url)
+        logger.info(f"Successfully exported FAQ page to Markdown: {url}")
+    else:
+        logger.error(f"Failed to crawl or export FAQ page: {url}")
+
+
 async def main():
     """Main function to run the FAQ detector."""
     config = FAQDetectorConfig()
@@ -327,4 +339,5 @@ async def main():
 
 
 if __name__ == "__main__":
+    # export_single_faq_page_to_md("url_to_faq_page_here")
     asyncio.run(main())

pages/ask_uos_chat.py

@@ -451,24 +451,36 @@ def _get_stream():
 
             except GraphRecursionError as e:
                 # TODO handle recursion limit error
-                logger.exception(f"[LANGGRAPH] Recursion Limit reached: {e}")
-                response = session_state["_"](
-                    "I'm sorry, but I couldn't find enough information to fully answer your question. Could you please try rephrasing your query and ask again?"
+                logger.warning(
+                    f"[NOT-ANSWERED] Recursion Limit reached. Query: {user_input} . Sources used. Documents: {graph._visited_docs()}, Urls: {graph._visited_links}"
+                )
+                response = (
+                    session_state["_"](
+                        "I'm sorry, but I couldn't find enough information to fully answer your question. Could you please try rephrasing your query and ask again?"
+                    )
+                    + f"\n\n{further_help_msg}"
                 )
-                st.markdown(f"{response}\n\n{further_help_msg}")
+
                 # clear the docs references
+                st.markdown(response)
                 graph._visited_docs.clear()
 
             except ProgrammableSearchException as e:
-                response = session_state["_"](
-                    "I'm sorry, something went wrong while connecting to the data provided. If the error persists, please reach out to the administrators for assistance."
+                response = (
+                    session_state["_"](
+                        "I'm sorry, something went wrong while connecting to the data provided. If the error persists, please reach out to the administrators for assistance."
+                    )
+                    + f"\n{further_help_msg}"
                 )
-                st.markdown(f"{response}\n{further_help_msg}")
+
+                st.markdown(response)
                 # clear the docs references
                 graph._visited_docs.clear()
 
             except Exception as e:
-                logger.exception(f"Error while processing the user's query: {e}")
+                logger.exception(
+                    f"[STREAMLIT] Error while processing the user's query: {e}"
+                )
                 response = session_state["_"](
                     "I'm sorry, but I am unable to process your request right now. Please try again later or consider rephrasing your question."
                 )
@@ -484,7 +496,15 @@ def _get_stream():
                 start_time = time.time()
                 settings.time_request_sent = start_time
 
-                response, to_stream = stream_graph_updates(prompt)
+                try:
+                    response, to_stream = stream_graph_updates(prompt)
+                except Exception as e:
+                    logger.error(f"[STREAMLIT] Error in streaming graph updates: {e}")
+                    response = session_state["_"](
+                        "I'm sorry, but I am unable to process your request right now. Please try again later or consider rephrasing your question."
+                    )
+                    to_stream = ""
+                    st.markdown(response)
                 # if there is content left, stream it
                 if to_stream:
                     st.markdown(to_stream)

src/chatbot_log/log_tags.md

@@ -20,6 +20,8 @@
 - VECTOR DB
 - RAGFlow
 - EMBEDDING
+- NOT-ANSWERED e.g., A recursion error occurs because the content needed to answer the question was not found. 
+- STREAMLIT
 
 
 ## Example Usage