From a99c823fb1b7a404ccc59b356409f0e4248b9c8d Mon Sep 17 00:00:00 2001 From: ma-zhongxu Date: Mon, 3 Mar 2025 19:10:08 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=87=E4=BB=B6=E4=B8=8A=E4=BC=A0=E8=B7=A8?= =?UTF-8?q?=E5=8D=B7=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 + collector/index.js | 89 +++++++++++++++++++++++++------ server/endpoints/workspaces.js | 96 ++++++++++++++++++++++++++++++++++ server/prisma/schema.prisma | 5 ++ 4 files changed, 176 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 9589fba..d179fac 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ v-env .env !.env.example .idea +/server/storage/documents +/server/storage/localFile node_modules __pycache__ diff --git a/collector/index.js b/collector/index.js index b307b58..99bdf24 100644 --- a/collector/index.js +++ b/collector/index.js @@ -28,33 +28,90 @@ app.use( }) ); +// app.post( +// "/process", +// [verifyPayloadIntegrity], +// async function (request, response) { +// const { filename, options = {} } = reqBody(request); +// try { +// const targetFilename = path +// .normalize(filename) +// .replace(/^(\.\.(\/|\\|$))+/, ""); +// const { +// success, +// reason, +// documents = [], +// } = await processSingleFile(targetFilename, options); +// response +// .status(200) +// .json({ filename: targetFilename, success, reason, documents }); +// } catch (e) { +// console.error(e); +// response.status(200).json({ +// filename: filename, +// success: false, +// reason: "A processing error occurred.", +// documents: [], +// }); +// } +// return; +// } +// ); + +const fs = require("fs").promises; // 使用 fs.promises 支持异步操作 +// const path = require("path"); + app.post( "/process", [verifyPayloadIntegrity], async function (request, response) { const { filename, options = {} } = reqBody(request); + console.log("文件名:", filename); + try { - const targetFilename = path - .normalize(filename) - .replace(/^(\.\.(\/|\\|$))+/, ""); - const { + const inputPath = path.resolve("./hotdir"); + const sourceFile = path.join(inputPath, filename); // 拼接文件路径 + console.log("源文件路径:", sourceFile); + + // 检查路径是否是文件 + const stats = await fs.stat(sourceFile); + if (!stats.isFile()) { + return response.status(400).json({ + success: false, + error: "提供的路径不是文件", + }); + } + + // 读取文件内容 + const fileContent = await fs.readFile(sourceFile); // 读取文件为 Buffer + const fileContentBase64 = fileContent.toString("base64"); // 将文件内容转换为 Base64 字符串 + + // 处理文件并返回结果 + const { success, reason, documents = [] } = await processSingleFile(sourceFile, options); + + response.status(200).json({ + filename: sourceFile, success, reason, - documents = [], - } = await processSingleFile(targetFilename, options); - response - .status(200) - .json({ filename: targetFilename, success, reason, documents }); + documents, + fileContent: fileContentBase64, // 将文件内容作为 Base64 字符串返回 + }); } catch (e) { console.error(e); - response.status(200).json({ - filename: filename, - success: false, - reason: "A processing error occurred.", - documents: [], - }); + if (e.code === "EISDIR") { + response.status(400).json({ + success: false, + error: "提供的路径是目录，不是文件", + }); + } else { + response.status(500).json({ + filename: filename, + success: false, + reason: "A processing error occurred.", + documents: [], + }); + } } - return; } ); diff --git a/server/endpoints/workspaces.js b/server/endpoints/workspaces.js index 16689dd..374a16a 100644 --- a/server/endpoints/workspaces.js +++ b/server/endpoints/workspaces.js @@ -244,6 +244,102 @@ function workspaceEndpoints(app) { } ); + + + app.post( + "/workspace/:slug/upload", + [ + validatedRequest, + flexUserRoleValid([ROLES.admin, ROLES.manager]), + handleFileUpload, + ], + async function (request, response) { + try { + const deptUserRecord = await DeptUsers.get({ userId: user.id }); + if (!deptUserRecord.deptUser) { + return response.status(500).json({ success: false, error: "没有发现用户组织机构" }); + } + const Collector = new CollectorApi(); + const { originalname } = request.file; + const processingOnline = await Collector.online(); + + if (!processingOnline) { + return response.status(500).json({ + success: false, + error: `Document processing API is not online. Document ${originalname} will not be processed automatically.`, + }); + } + + // 处理文档 + const { success, reason, documents, fileContent } = + await Collector.processDocument(originalname); + if (!success) { + return response.status(500).json({ success: false, error: reason }); + } + + // 确定目标目录 + const targetDir = + process.env.NODE_ENV === "development" + ? path.resolve(__dirname, "../../server/storage/localFile") + : path.resolve(process.env.STORAGE_DIR, "localFile"); + + // 确保目标目录存在 + if (!fs.existsSync(targetDir)) { + fs.mkdirSync(targetDir, { recursive: true }); // 递归创建目录 + } + + // 保存文件 + const filePath = path.join(targetDir, originalname); // 使用原始文件名 + const fileBuffer = Buffer.from(fileContent, "base64"); + fs.writeFileSync(filePath, fileBuffer); + + console.log(`文件保存成功！路径：${filePath}`); + // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + // 假设路径字符串 + const location = documents[0].location; + // 将路径中的反斜杠替换为正斜杠（可选，但通常更通用） + const unixStylePath = location.replace(/\\/g, '/'); + // 找到最后一个目录分隔符的位置 + const lastIndex = unixStylePath.lastIndexOf('/'); + // 提取文件名 + const parsedFileName = unixStylePath.substring(lastIndex + 1); + const fileExtension = path.extname(request.file.path).toLowerCase(); + const sourceFile = path.resolve(__dirname, request.file.destination, request.file.originalname); + const newFileName = uuidv4() + fileExtension; // 新文件名 + moveAndRenameFile(sourceFile, targetDir, newFileName); + const deptDocData = { + deptId: deptUserRecord.deptUser.deptId, + parsedFileName: parsedFileName, + parsedFilePath: location, + realFileName: originalname, + realFileAlias: newFileName, + realFilePath: targetDir, + }; + await DeptDocument.create(deptDocData); + // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + // 记录日志和发送遥测 + Collector.log( + `Document ${originalname} uploaded, processed, and saved successfully. It is now available in documents.` + ); + await Telemetry.sendTelemetry("document_uploaded"); + await EventLogs.logEvent( + "document_uploaded", + { + documentName: originalname, + }, + response.locals?.user?.id + ); + + // 返回成功响应 + response.status(200).json({ success: true, error: null }); + } catch (e) { + console.error(e.message, e); + response.status(500).json({ success: false, error: e.message }); + } + } + ); + app.post( "/workspace/:slug/upload-link", [validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])], diff --git a/server/prisma/schema.prisma b/server/prisma/schema.prisma index 30eaecb..411bda3 100644 --- a/server/prisma/schema.prisma +++ b/server/prisma/schema.prisma @@ -7,6 +7,11 @@ datasource db { url = "file:../storage/anythingllm.db" } +// datasource db { +// provider = "mysql" +// url = env("DATABASE_URL") +// } + model api_keys { id Int @id @default(autoincrement()) secret String? @unique