完善后端日志
This commit is contained in:
@@ -23,6 +23,8 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.core.database.mysql import Base, mysql_db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
# 设置该模块的日志级别
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
class ExcelStorageService:
|
||||
@@ -174,12 +176,15 @@ class ExcelStorageService:
|
||||
}
|
||||
|
||||
try:
|
||||
logger.info(f"开始读取Excel文件: {file_path}")
|
||||
# 读取 Excel
|
||||
if sheet_name:
|
||||
df = pd.read_excel(file_path, sheet_name=sheet_name, header=header_row)
|
||||
else:
|
||||
df = pd.read_excel(file_path, header=header_row)
|
||||
|
||||
logger.info(f"Excel读取完成,行数: {len(df)}, 列数: {len(df.columns)}")
|
||||
|
||||
if df.empty:
|
||||
return {"success": False, "error": "Excel 文件为空"}
|
||||
|
||||
@@ -202,8 +207,10 @@ class ExcelStorageService:
|
||||
model_class = self._create_table_model(table_name, df.columns, column_types)
|
||||
|
||||
# 创建表结构
|
||||
logger.info(f"正在创建MySQL表: {table_name}")
|
||||
async with self.mysql_db.get_session() as session:
|
||||
model_class.__table__.create(session.bind, checkfirst=True)
|
||||
logger.info(f"MySQL表创建完成: {table_name}")
|
||||
|
||||
# 插入数据
|
||||
records = []
|
||||
@@ -231,11 +238,13 @@ class ExcelStorageService:
|
||||
|
||||
records.append(record)
|
||||
|
||||
logger.info(f"正在插入 {len(records)} 条数据到 MySQL...")
|
||||
# 批量插入
|
||||
async with self.mysql_db.get_session() as session:
|
||||
for record in records:
|
||||
session.add(model_class(**record))
|
||||
await session.commit()
|
||||
logger.info(f"数据插入完成: {len(records)} 条")
|
||||
|
||||
results["row_count"] = len(records)
|
||||
logger.info(f"Excel 数据已存储到 MySQL 表 {table_name},共 {len(records)} 行")
|
||||
@@ -243,7 +252,7 @@ class ExcelStorageService:
|
||||
return results
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"存储 Excel 到 MySQL 失败: {str(e)}")
|
||||
logger.error(f"存储 Excel 到 MySQL 失败: {str(e)}", exc_info=True)
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
async def store_structured_data(
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
@@ -10,6 +11,8 @@ import uuid
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FileService:
|
||||
"""文件服务类,负责文件的存储、读取和管理"""
|
||||
@@ -17,6 +20,7 @@ class FileService:
|
||||
def __init__(self):
|
||||
self.upload_dir = Path(settings.UPLOAD_DIR)
|
||||
self._ensure_upload_dir()
|
||||
logger.info(f"FileService 初始化,上传目录: {self.upload_dir}")
|
||||
|
||||
def _ensure_upload_dir(self):
|
||||
"""确保上传目录存在"""
|
||||
@@ -56,6 +60,8 @@ class FileService:
|
||||
with open(file_path, 'wb') as f:
|
||||
f.write(file_content)
|
||||
|
||||
file_size = len(file_content)
|
||||
logger.info(f"文件已保存: {filename} -> {file_path} ({file_size} bytes)")
|
||||
return str(file_path)
|
||||
|
||||
def read_file(self, file_path: str) -> bytes:
|
||||
|
||||
@@ -126,26 +126,45 @@ class TableRAGService:
|
||||
}
|
||||
|
||||
try:
|
||||
# 1. 读取 Excel
|
||||
# 1. 先检查 Excel 文件是否有效
|
||||
logger.info(f"正在检查Excel文件: {file_path}")
|
||||
try:
|
||||
xls_file = pd.ExcelFile(file_path)
|
||||
sheet_names = xls_file.sheet_names
|
||||
logger.info(f"Excel文件工作表: {sheet_names}")
|
||||
if not sheet_names:
|
||||
return {"success": False, "error": "Excel 文件没有工作表"}
|
||||
except Exception as e:
|
||||
logger.error(f"读取Excel文件失败: {file_path}, error: {e}")
|
||||
return {"success": False, "error": f"无法读取Excel文件: {str(e)}"}
|
||||
|
||||
# 2. 读取 Excel
|
||||
if sheet_name:
|
||||
# 验证指定的sheet_name是否存在
|
||||
if sheet_name not in sheet_names:
|
||||
logger.warning(f"指定的工作表 '{sheet_name}' 不存在,使用第一个工作表: {sheet_names[0]}")
|
||||
sheet_name = sheet_names[0]
|
||||
df = pd.read_excel(file_path, sheet_name=sheet_name, header=header_row)
|
||||
else:
|
||||
df = pd.read_excel(file_path, header=header_row)
|
||||
|
||||
logger.info(f"读取到数据: {len(df)} 行, {len(df.columns)} 列")
|
||||
|
||||
if df.empty:
|
||||
return {"success": False, "error": "Excel 文件为空"}
|
||||
|
||||
# 清理列名
|
||||
df.columns = [str(c) for c in df.columns]
|
||||
table_name = excel_storage._sanitize_table_name(filename)
|
||||
table_name = self.excel_storage._sanitize_table_name(filename)
|
||||
results["table_name"] = table_name
|
||||
results["field_count"] = len(df.columns)
|
||||
logger.info(f"表名: {table_name}, 字段数: {len(df.columns)}")
|
||||
|
||||
# 2. 初始化 RAG (如果需要)
|
||||
# 3. 初始化 RAG (如果需要)
|
||||
if not self.rag._initialized:
|
||||
self.rag._init_vector_store()
|
||||
|
||||
# 3. 为每个字段生成描述并索引
|
||||
# 4. 为每个字段生成描述并索引
|
||||
all_fields_data = {}
|
||||
for col in df.columns:
|
||||
# 采样示例值
|
||||
@@ -187,7 +206,8 @@ class TableRAGService:
|
||||
logger.error(error_msg)
|
||||
results["errors"].append(error_msg)
|
||||
|
||||
# 4. 存储到 MySQL
|
||||
# 5. 存储到 MySQL
|
||||
logger.info(f"开始存储到MySQL: {filename}")
|
||||
store_result = await self.excel_storage.store_excel(
|
||||
file_path=file_path,
|
||||
filename=filename,
|
||||
|
||||
Reference in New Issue
Block a user