Merge branch 'main' of https://gitea.kronecker.cc/OurCodesAreAllRight/FilesReadSystem

backend/app/api/endpoints/ai_analyze.py

@@ -216,9 +216,12 @@ async def analyze_markdown(
             return result
 
         finally:
-            # 清理临时文件
-            if os.path.exists(tmp_path):
-                os.unlink(tmp_path)
+            # 清理临时文件，确保在所有情况下都能清理
+            try:
+                if tmp_path and os.path.exists(tmp_path):
+                    os.unlink(tmp_path)
+            except Exception as cleanup_error:
+                logger.warning(f"临时文件清理失败: {tmp_path}, error: {cleanup_error}")
 
     except HTTPException:
         raise
@@ -280,8 +283,12 @@ async def analyze_markdown_stream(
             )
 
         finally:
-            if os.path.exists(tmp_path):
-                os.unlink(tmp_path)
+            # 清理临时文件，确保在所有情况下都能清理
+            try:
+                if tmp_path and os.path.exists(tmp_path):
+                    os.unlink(tmp_path)
+            except Exception as cleanup_error:
+                logger.warning(f"临时文件清理失败: {tmp_path}, error: {cleanup_error}")
 
     except HTTPException:
         raise
@@ -290,7 +297,7 @@ async def analyze_markdown_stream(
         raise HTTPException(status_code=500, detail=f"流式分析失败: {str(e)}")
 
 
-@router.get("/analyze/md/outline")
+@router.post("/analyze/md/outline")
 async def get_markdown_outline(
     file: UploadFile = File(...)
 ):
@@ -324,8 +331,12 @@ async def get_markdown_outline(
             result = await markdown_ai_service.extract_outline(tmp_path)
             return result
         finally:
-            if os.path.exists(tmp_path):
-                os.unlink(tmp_path)
+            # 清理临时文件，确保在所有情况下都能清理
+            try:
+                if tmp_path and os.path.exists(tmp_path):
+                    os.unlink(tmp_path)
+            except Exception as cleanup_error:
+                logger.warning(f"临时文件清理失败: {tmp_path}, error: {cleanup_error}")
 
     except Exception as e:
         logger.error(f"获取 Markdown 大纲失败: {str(e)}")

backend/app/api/endpoints/documents.py

@@ -23,6 +23,52 @@ logger = logging.getLogger(__name__)
 router = APIRouter(prefix="/upload", tags=["文档上传"])
 
 
+# ==================== 辅助函数 ====================
+
+async def update_task_status(
+    task_id: str,
+    status: str,
+    progress: int = 0,
+    message: str = "",
+    result: dict = None,
+    error: str = None
+):
+    """
+    更新任务状态，同时写入 Redis 和 MongoDB
+
+    Args:
+        task_id: 任务ID
+        status: 状态
+        progress: 进度
+        message: 消息
+        result: 结果
+        error: 错误信息
+    """
+    meta = {"progress": progress, "message": message}
+    if result:
+        meta["result"] = result
+    if error:
+        meta["error"] = error
+
+    # 尝试写入 Redis
+    try:
+        await redis_db.set_task_status(task_id, status, meta)
+    except Exception as e:
+        logger.warning(f"Redis 任务状态更新失败: {e}")
+
+    # 尝试写入 MongoDB（作为备用）
+    try:
+        await mongodb.update_task(
+            task_id=task_id,
+            status=status,
+            message=message,
+            result=result,
+            error=error
+        )
+    except Exception as e:
+        logger.warning(f"MongoDB 任务状态更新失败: {e}")
+
+
 # ==================== 请求/响应模型 ====================
 
 class UploadResponse(BaseModel):
@@ -77,6 +123,17 @@ async def upload_document(
     task_id = str(uuid.uuid4())
 
     try:
+        # 保存任务记录到 MongoDB（如果 Redis 不可用时仍能查询）
+        try:
+            await mongodb.insert_task(
+                task_id=task_id,
+                task_type="document_parse",
+                status="pending",
+                message=f"文档 {file.filename} 已提交处理"
+            )
+        except Exception as mongo_err:
+            logger.warning(f"MongoDB 保存任务记录失败: {mongo_err}")
+
         content = await file.read()
         saved_path = file_service.save_uploaded_file(
             content,
@@ -122,6 +179,17 @@ async def upload_documents(
     saved_paths = []
 
     try:
+        # 保存任务记录到 MongoDB
+        try:
+            await mongodb.insert_task(
+                task_id=task_id,
+                task_type="batch_parse",
+                status="pending",
+                message=f"已提交 {len(files)} 个文档处理"
+            )
+        except Exception as mongo_err:
+            logger.warning(f"MongoDB 保存批量任务记录失败: {mongo_err}")
+
         for file in files:
             if not file.filename:
                 continue
@@ -159,9 +227,9 @@ async def process_document(
     """处理单个文档"""
     try:
         # 状态: 解析中
-        await redis_db.set_task_status(
+        await update_task_status(
             task_id, status="processing",
-            meta={"progress": 10, "message": "正在解析文档"}
+            progress=10, message="正在解析文档"
         )
 
         # 解析文档
@@ -172,9 +240,9 @@ async def process_document(
             raise Exception(result.error or "解析失败")
 
         # 状态: 存储中
-        await redis_db.set_task_status(
+        await update_task_status(
             task_id, status="processing",
-            meta={"progress": 30, "message": "正在存储数据"}
+            progress=30, message="正在存储数据"
         )
 
         # 存储到 MongoDB
@@ -235,9 +303,9 @@ async def process_document(
 
         # 如果是 Excel，存储到 MySQL + AI生成描述 + RAG索引
         if doc_type in ["xlsx", "xls"]:
-            await redis_db.set_task_status(
+            await update_task_status(
                 task_id, status="processing",
-                meta={"progress": 50, "message": "正在存储到MySQL并生成字段描述"}
+                progress=50, message="正在存储到MySQL并生成字段描述"
             )
 
             try:
@@ -259,9 +327,9 @@ async def process_document(
 
         else:
             # 非结构化文档
-            await redis_db.set_task_status(
+            await update_task_status(
                 task_id, status="processing",
-                meta={"progress": 60, "message": "正在建立索引"}
+                progress=60, message="正在建立索引"
             )
 
             # 如果文档中有表格数据，提取并存储到 MySQL + RAG
@@ -282,17 +350,13 @@ async def process_document(
             await index_document_to_rag(doc_id, original_filename, result, doc_type)
 
         # 完成
-        await redis_db.set_task_status(
+        await update_task_status(
             task_id, status="success",
-            meta={
-                "progress": 100,
-                "message": "处理完成",
+            progress=100, message="处理完成",
+            result={
                 "doc_id": doc_id,
-                "result": {
-                    "doc_id": doc_id,
-                    "doc_type": doc_type,
-                    "filename": original_filename
-                }
+                "doc_type": doc_type,
+                "filename": original_filename
             }
         )
 
@@ -300,18 +364,19 @@ async def process_document(
 
     except Exception as e:
         logger.error(f"文档处理失败: {str(e)}")
-        await redis_db.set_task_status(
+        await update_task_status(
             task_id, status="failure",
-            meta={"error": str(e)}
+            progress=0, message="处理失败",
+            error=str(e)
         )
 
 
 async def process_documents_batch(task_id: str, files: List[dict]):
     """批量处理文档"""
     try:
-        await redis_db.set_task_status(
+        await update_task_status(
             task_id, status="processing",
-            meta={"progress": 0, "message": "开始批量处理"}
+            progress=0, message="开始批量处理"
         )
 
         results = []
@@ -362,21 +427,23 @@ async def process_documents_batch(task_id: str, files: List[dict]):
                 results.append({"filename": file_info["filename"], "success": False, "error": str(e)})
 
             progress = int((i + 1) / len(files) * 100)
-            await redis_db.set_task_status(
+            await update_task_status(
                 task_id, status="processing",
-                meta={"progress": progress, "message": f"已处理 {i+1}/{len(files)}"}
+                progress=progress, message=f"已处理 {i+1}/{len(files)}"
             )
 
-        await redis_db.set_task_status(
+        await update_task_status(
             task_id, status="success",
-            meta={"progress": 100, "message": "批量处理完成", "results": results}
+            progress=100, message="批量处理完成",
+            result={"results": results}
         )
 
     except Exception as e:
         logger.error(f"批量处理失败: {str(e)}")
-        await redis_db.set_task_status(
+        await update_task_status(
             task_id, status="failure",
-            meta={"error": str(e)}
+            progress=0, message="批量处理失败",
+            error=str(e)
         )
 
 

backend/app/api/endpoints/health.py

@@ -19,26 +19,43 @@ async def health_check() -> Dict[str, Any]:
     返回各数据库连接状态和应用信息
     """
     # 检查各数据库连接状态
-    mysql_status = "connected"
-    mongodb_status = "connected"
-    redis_status = "connected"
+    mysql_status = "unknown"
+    mongodb_status = "unknown"
+    redis_status = "unknown"
 
     try:
         if mysql_db.async_engine is None:
             mysql_status = "disconnected"
-    except Exception:
+        else:
+            # 实际执行一次查询验证连接
+            from sqlalchemy import text
+            async with mysql_db.async_engine.connect() as conn:
+                await conn.execute(text("SELECT 1"))
+            mysql_status = "connected"
+    except Exception as e:
+        logger.warning(f"MySQL 健康检查失败: {e}")
         mysql_status = "error"
 
     try:
         if mongodb.client is None:
             mongodb_status = "disconnected"
-    except Exception:
+        else:
+            # 实际 ping 验证
+            await mongodb.client.admin.command('ping')
+            mongodb_status = "connected"
+    except Exception as e:
+        logger.warning(f"MongoDB 健康检查失败: {e}")
         mongodb_status = "error"
 
     try:
-        if not redis_db.is_connected:
+        if not redis_db.is_connected or redis_db.client is None:
             redis_status = "disconnected"
-    except Exception:
+        else:
+            # 实际执行 ping 验证
+            await redis_db.client.ping()
+            redis_status = "connected"
+    except Exception as e:
+        logger.warning(f"Redis 健康检查失败: {e}")
         redis_status = "error"
 
     return {

backend/app/api/endpoints/tasks.py

@@ -1,13 +1,13 @@
 """
 任务管理 API 接口
 
-提供异步任务状态查询
+提供异步任务状态查询和历史记录
 """
 from typing import Optional
 
 from fastapi import APIRouter, HTTPException
 
-from app.core.database import redis_db
+from app.core.database import redis_db, mongodb
 
 router = APIRouter(prefix="/tasks", tags=["任务管理"])
 
@@ -23,25 +23,94 @@ async def get_task_status(task_id: str):
     Returns:
         任务状态信息
     """
+    # 优先从 Redis 获取
     status = await redis_db.get_task_status(task_id)
 
-    if not status:
-        # Redis不可用时，假设任务已完成（文档已成功处理）
-        # 前端轮询时会得到这个响应
+    if status:
         return {
             "task_id": task_id,
-            "status": "success",
-            "progress": 100,
-            "message": "任务处理完成",
-            "result": None,
-            "error": None
+            "status": status.get("status", "unknown"),
+            "progress": status.get("meta", {}).get("progress", 0),
+            "message": status.get("meta", {}).get("message"),
+            "result": status.get("meta", {}).get("result"),
+            "error": status.get("meta", {}).get("error")
         }
 
+    # Redis 不可用时，尝试从 MongoDB 获取
+    mongo_task = await mongodb.get_task(task_id)
+    if mongo_task:
+        return {
+            "task_id": mongo_task.get("task_id"),
+            "status": mongo_task.get("status", "unknown"),
+            "progress": 100 if mongo_task.get("status") == "success" else 0,
+            "message": mongo_task.get("message"),
+            "result": mongo_task.get("result"),
+            "error": mongo_task.get("error")
+        }
+
+    # 任务不存在或状态未知
     return {
         "task_id": task_id,
-        "status": status.get("status", "unknown"),
-        "progress": status.get("meta", {}).get("progress", 0),
-        "message": status.get("meta", {}).get("message"),
-        "result": status.get("meta", {}).get("result"),
-        "error": status.get("meta", {}).get("error")
+        "status": "unknown",
+        "progress": 0,
+        "message": "无法获取任务状态（Redis和MongoDB均不可用）",
+        "result": None,
+        "error": None
     }
+
+
+@router.get("/")
+async def list_tasks(limit: int = 50, skip: int = 0):
+    """
+    获取任务历史列表
+
+    Args:
+        limit: 返回数量限制
+        skip: 跳过数量
+
+    Returns:
+        任务列表
+    """
+    try:
+        tasks = await mongodb.list_tasks(limit=limit, skip=skip)
+        return {
+            "success": True,
+            "tasks": tasks,
+            "count": len(tasks)
+        }
+    except Exception as e:
+        # MongoDB 不可用时返回空列表
+        return {
+            "success": False,
+            "tasks": [],
+            "count": 0,
+            "error": str(e)
+        }
+
+
+@router.delete("/{task_id}")
+async def delete_task(task_id: str):
+    """
+    删除任务
+
+    Args:
+        task_id: 任务ID
+
+    Returns:
+        是否删除成功
+    """
+    try:
+        # 从 Redis 删除
+        if redis_db._connected and redis_db.client:
+            key = f"task:{task_id}"
+            await redis_db.client.delete(key)
+
+        # 从 MongoDB 删除
+        deleted = await mongodb.delete_task(task_id)
+
+        return {
+            "success": True,
+            "deleted": deleted
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"删除任务失败: {str(e)}")

backend/app/api/endpoints/templates.py

@@ -23,6 +23,44 @@ logger = logging.getLogger(__name__)
 router = APIRouter(prefix="/templates", tags=["表格模板"])
 
 
+# ==================== 辅助函数 ====================
+
+async def update_task_status(
+    task_id: str,
+    status: str,
+    progress: int = 0,
+    message: str = "",
+    result: dict = None,
+    error: str = None
+):
+    """
+    更新任务状态，同时写入 Redis 和 MongoDB
+    """
+    from app.core.database import redis_db
+
+    meta = {"progress": progress, "message": message}
+    if result:
+        meta["result"] = result
+    if error:
+        meta["error"] = error
+
+    try:
+        await redis_db.set_task_status(task_id, status, meta)
+    except Exception as e:
+        logger.warning(f"Redis 任务状态更新失败: {e}")
+
+    try:
+        await mongodb.update_task(
+            task_id=task_id,
+            status=status,
+            message=message,
+            result=result,
+            error=error
+        )
+    except Exception as e:
+        logger.warning(f"MongoDB 任务状态更新失败: {e}")
+
+
 # ==================== 请求/响应模型 ====================
 
 class TemplateFieldRequest(BaseModel):
@@ -41,6 +79,7 @@ class FillRequest(BaseModel):
     source_doc_ids: Optional[List[str]] = None  # MongoDB 文档 ID 列表
     source_file_paths: Optional[List[str]] = None  # 源文档文件路径列表
     user_hint: Optional[str] = None
+    task_id: Optional[str] = None  # 可选的任务ID，用于任务历史跟踪
 
 
 class ExportRequest(BaseModel):
@@ -162,20 +201,17 @@ async def upload_joint_template(
                 )
 
     try:
-        # 1. 保存模板文件并提取字段
+        # 1. 保存模板文件
         template_content = await template_file.read()
         template_path = file_service.save_uploaded_file(
             template_content,
             template_file.filename,
             subfolder="templates"
         )
-        template_fields = await template_fill_service.get_template_fields_from_file(
-            template_path,
-            template_ext
-        )
 
-        # 2. 处理源文档 - 保存文件
+        # 2. 保存并解析源文档 - 提取内容用于生成表头
         source_file_info = []
+        source_contents = []
         for sf in source_files:
             if sf.filename:
                 sf_content = await sf.read()
@@ -190,10 +226,81 @@ async def upload_joint_template(
                     "filename": sf.filename,
                     "ext": sf_ext
                 })
+                # 解析源文档获取内容（用于 AI 生成表头）
+                try:
+                    from app.core.document_parser import ParserFactory
+                    parser = ParserFactory.get_parser(sf_path)
+                    parse_result = parser.parse(sf_path)
+                    if parse_result.success and parse_result.data:
+                        # 获取原始内容
+                        content = parse_result.data.get("content", "")[:5000] if parse_result.data.get("content") else ""
+
+                        # 获取标题（可能在顶层或structured_data内）
+                        titles = parse_result.data.get("titles", [])
+                        if not titles and parse_result.data.get("structured_data"):
+                            titles = parse_result.data.get("structured_data", {}).get("titles", [])
+                        titles = titles[:10] if titles else []
+
+                        # 获取表格数量（可能在顶层或structured_data内）
+                        tables = parse_result.data.get("tables", [])
+                        if not tables and parse_result.data.get("structured_data"):
+                            tables = parse_result.data.get("structured_data", {}).get("tables", [])
+                        tables_count = len(tables) if tables else 0
+
+                        # 获取表格内容摘要（用于 AI 理解源文档结构）
+                        tables_summary = ""
+                        if tables:
+                            tables_summary = "\n【文档中的表格】:\n"
+                            for idx, table in enumerate(tables[:5]):  # 最多5个表格
+                                if isinstance(table, dict):
+                                    headers = table.get("headers", [])
+                                    rows = table.get("rows", [])
+                                    if headers:
+                                        tables_summary += f"表格{idx+1}表头: {', '.join(str(h) for h in headers)}\n"
+                                    if rows:
+                                        tables_summary += f"表格{idx+1}前3行: "
+                                        for row_idx, row in enumerate(rows[:3]):
+                                            if isinstance(row, list):
+                                                tables_summary += " | ".join(str(c) for c in row) + "; "
+                                            elif isinstance(row, dict):
+                                                tables_summary += " | ".join(str(row.get(h, "")) for h in headers if headers) + "; "
+                                        tables_summary += "\n"
+
+                        source_contents.append({
+                            "filename": sf.filename,
+                            "doc_type": sf_ext,
+                            "content": content,
+                            "titles": titles,
+                            "tables_count": tables_count,
+                            "tables_summary": tables_summary
+                        })
+                        logger.info(f"[DEBUG] source_contents built: filename={sf.filename}, content_len={len(content)}, titles_count={len(titles)}, tables_count={tables_count}")
+                        if tables_summary:
+                            logger.info(f"[DEBUG] tables_summary preview: {tables_summary[:300]}")
+                except Exception as e:
+                    logger.warning(f"解析源文档失败 {sf.filename}: {e}")
+
+        # 3. 根据源文档内容生成表头
+        template_fields = await template_fill_service.get_template_fields_from_file(
+            template_path,
+            template_ext,
+            source_contents=source_contents  # 传递源文档内容
+        )
 
         # 3. 异步处理源文档到MongoDB
         task_id = str(uuid.uuid4())
         if source_file_info:
+            # 保存任务记录到 MongoDB
+            try:
+                await mongodb.insert_task(
+                    task_id=task_id,
+                    task_type="source_process",
+                    status="pending",
+                    message=f"开始处理 {len(source_file_info)} 个源文档"
+                )
+            except Exception as mongo_err:
+                logger.warning(f"MongoDB 保存任务记录失败: {mongo_err}")
+
             background_tasks.add_task(
                 process_source_documents,
                 task_id=task_id,
@@ -232,12 +339,10 @@ async def upload_joint_template(
 
 async def process_source_documents(task_id: str, files: List[dict]):
     """异步处理源文档，存入MongoDB"""
-    from app.core.database import redis_db
-
     try:
-        await redis_db.set_task_status(
+        await update_task_status(
             task_id, status="processing",
-            meta={"progress": 0, "message": "开始处理源文档"}
+            progress=0, message="开始处理源文档"
         )
 
         doc_ids = []
@@ -266,22 +371,24 @@ async def process_source_documents(task_id: str, files: List[dict]):
                 logger.error(f"源文档处理异常: {file_info['filename']}, error: {str(e)}")
 
             progress = int((i + 1) / len(files) * 100)
-            await redis_db.set_task_status(
+            await update_task_status(
                 task_id, status="processing",
-                meta={"progress": progress, "message": f"已处理 {i+1}/{len(files)}"}
+                progress=progress, message=f"已处理 {i+1}/{len(files)}"
             )
 
-        await redis_db.set_task_status(
+        await update_task_status(
             task_id, status="success",
-            meta={"progress": 100, "message": "源文档处理完成", "doc_ids": doc_ids}
+            progress=100, message="源文档处理完成",
+            result={"doc_ids": doc_ids}
         )
         logger.info(f"所有源文档处理完成: {len(doc_ids)}个")
 
     except Exception as e:
         logger.error(f"源文档批量处理失败: {str(e)}")
-        await redis_db.set_task_status(
+        await update_task_status(
             task_id, status="failure",
-            meta={"error": str(e)}
+            progress=0, message="源文档处理失败",
+            error=str(e)
         )
 
 
@@ -340,7 +447,27 @@ async def fill_template(
     Returns:
         填写结果
     """
+    # 生成或使用传入的 task_id
+    task_id = request.task_id or str(uuid.uuid4())
+
     try:
+        # 创建任务记录到 MongoDB
+        try:
+            await mongodb.insert_task(
+                task_id=task_id,
+                task_type="template_fill",
+                status="processing",
+                message=f"开始填表任务: {len(request.template_fields)} 个字段"
+            )
+        except Exception as mongo_err:
+            logger.warning(f"MongoDB 创建任务记录失败: {mongo_err}")
+
+        # 更新进度 - 开始
+        await update_task_status(
+            task_id, "processing",
+            progress=0, message="开始处理..."
+        )
+
         # 转换字段
         fields = [
             TemplateField(
@@ -353,17 +480,51 @@ async def fill_template(
             for f in request.template_fields
         ]
 
+        # 从 template_id 提取文件类型
+        template_file_type = "xlsx"  # 默认类型
+        if request.template_id:
+            ext = request.template_id.split('.')[-1].lower()
+            if ext in ["xlsx", "xls"]:
+                template_file_type = "xlsx"
+            elif ext == "docx":
+                template_file_type = "docx"
+
+        # 更新进度 - 准备开始填写
+        await update_task_status(
+            task_id, "processing",
+            progress=10, message=f"准备填写 {len(fields)} 个字段..."
+        )
+
         # 执行填写
         result = await template_fill_service.fill_template(
             template_fields=fields,
             source_doc_ids=request.source_doc_ids,
             source_file_paths=request.source_file_paths,
-            user_hint=request.user_hint
+            user_hint=request.user_hint,
+            template_id=request.template_id,
+            template_file_type=template_file_type,
+            task_id=task_id
         )
 
-        return result
+        # 更新为成功
+        await update_task_status(
+            task_id, "success",
+            progress=100, message="填表完成",
+            result={
+                "field_count": len(fields),
+                "max_rows": result.get("max_rows", 0)
+            }
+        )
+
+        return {**result, "task_id": task_id}
 
     except Exception as e:
+        # 更新为失败
+        await update_task_status(
+            task_id, "failure",
+            progress=0, message="填表失败",
+            error=str(e)
+        )
         logger.error(f"填写表格失败: {str(e)}")
         raise HTTPException(status_code=500, detail=f"填写失败: {str(e)}")
 

backend/app/api/endpoints/upload.py

@@ -5,6 +5,7 @@ from fastapi import APIRouter, UploadFile, File, HTTPException, Query
 from fastapi.responses import StreamingResponse
 from typing import Optional
 import logging
+import os
 import pandas as pd
 import io
 
@@ -126,7 +127,7 @@ async def upload_excel(
                                 content += f"... (共 {len(sheet_data['rows'])} 行)\n\n"
 
             doc_metadata = {
-                "filename": saved_path.split("/")[-1] if "/" in saved_path else saved_path.split("\\")[-1],
+                "filename": os.path.basename(saved_path),
                 "original_filename": file.filename,
                 "saved_path": saved_path,
                 "file_size": len(content),
@@ -253,7 +254,7 @@ async def export_excel(
         output.seek(0)
 
         # 生成文件名
-        original_name = file_path.split('/')[-1] if '/' in file_path else file_path
+        original_name = os.path.basename(file_path)
         if columns:
             export_name = f"export_{sheet_name or 'data'}_{len(column_list) if columns else 'all'}_cols.xlsx"
         else: