|
|
@@ -0,0 +1,262 @@
|
|
|
+// import pdf from 'pdf-parse';
|
|
|
+// import fs from 'fs';
|
|
|
+import { CloudApi, CloudObject, CloudQuery } from 'src/lib/ncloud';
|
|
|
+import mammoth from "mammoth";
|
|
|
+import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
|
|
|
+import { Document } from '@langchain/core/documents';
|
|
|
+
|
|
|
+import * as tf from '@tensorflow/tfjs-core';
|
|
|
+// import "@tensorflow/tfjs-backend-cpu";
|
|
|
+// import '@tensorflow/tfjs-backend-webgpu';
|
|
|
+import '@tensorflow/tfjs-backend-webgl';
|
|
|
+// import '@tensorflow/tfjs-backend-wasm';
|
|
|
+import { TensorFlowEmbeddings } from "@langchain/community/embeddings/tensorflow";
|
|
|
+
|
|
|
+export class AgentStory{
|
|
|
+
|
|
|
+ story:CloudObject|undefined
|
|
|
+ // 文件标题
|
|
|
+ title:string|undefined = ""
|
|
|
+ // 文档标签
|
|
|
+ tags:Array<string>|undefined
|
|
|
+ // 文件源地址
|
|
|
+ url:string|undefined = ""
|
|
|
+ // 文档完整纯文本内容
|
|
|
+ content:string|undefined = ""
|
|
|
+ // 文档hash唯一值
|
|
|
+ hash:string|undefined = ""
|
|
|
+ // 文档分割后的列表
|
|
|
+ docList:Array<Document|any> = []
|
|
|
+
|
|
|
+ constructor(metadata:{
|
|
|
+ url:string,
|
|
|
+ title?:string,
|
|
|
+ tags?:Array<string>
|
|
|
+ }){
|
|
|
+ this.url = metadata.url
|
|
|
+ this.title = metadata.title
|
|
|
+ this.tags = metadata.tags
|
|
|
+ setBackend()
|
|
|
+ }
|
|
|
+ async save(){
|
|
|
+ if(!this.hash){ return }
|
|
|
+ let query = new CloudQuery("Story");
|
|
|
+ query.equalTo("hash",this.hash);
|
|
|
+ let story = await query.first();
|
|
|
+ if(!story?.id){
|
|
|
+ story = new CloudObject("Story");
|
|
|
+ }
|
|
|
+ story.set({
|
|
|
+ title: this.title,
|
|
|
+ url: this.url,
|
|
|
+ content: this.content,
|
|
|
+ hash: this.hash,
|
|
|
+ tags:this.tags
|
|
|
+ })
|
|
|
+ this.story = await story.save();
|
|
|
+ }
|
|
|
+ async loader(url:string){
|
|
|
+ let api = new CloudApi();
|
|
|
+
|
|
|
+ let result;
|
|
|
+ if(url?.endsWith(".docx")){
|
|
|
+ result = await this.loadDocx(url)
|
|
|
+ }
|
|
|
+ if(!result){
|
|
|
+ result = await api.fetch("agent/loader",{url:url})
|
|
|
+ }
|
|
|
+ this.content = result?.data || null
|
|
|
+ if(this.content){
|
|
|
+ this.url = url
|
|
|
+ }
|
|
|
+ this.save();
|
|
|
+ return this.content
|
|
|
+ }
|
|
|
+
|
|
|
+ async loadDocx(url:string){
|
|
|
+ let data:any
|
|
|
+ const response = await fetch(url);
|
|
|
+
|
|
|
+ const arrayBuffer:any = await response.arrayBuffer();
|
|
|
+
|
|
|
+ let text;
|
|
|
+ try {
|
|
|
+ text = await mammoth.extractRawText({arrayBuffer:arrayBuffer}); // 浏览器 直接传递 arrayBuffer
|
|
|
+ } catch (err) {
|
|
|
+ console.error(err);
|
|
|
+ }
|
|
|
+
|
|
|
+ this.hash = await arrayBufferToHASH(arrayBuffer)
|
|
|
+
|
|
|
+ // let html = mammoth.convertToHtml(buffer)
|
|
|
+ data = text?.value || "";
|
|
|
+ // 正则匹配所有 多个\n换行的字符 替换成一次换行
|
|
|
+ data = data.replaceAll(/\n+/g,"\n") // 剔除多余换行
|
|
|
+ return {data}
|
|
|
+ }
|
|
|
+ async splitter(options?:{
|
|
|
+ chunkSize:number,
|
|
|
+ chunkOverlap:number
|
|
|
+ }){
|
|
|
+ if(!this.content) return
|
|
|
+ // 默认：递归字符文本分割器
|
|
|
+ let splitter = new RecursiveCharacterTextSplitter({
|
|
|
+ chunkSize: options?.chunkSize || 500,
|
|
|
+ chunkOverlap: options?.chunkOverlap || 150,
|
|
|
+ });
|
|
|
+
|
|
|
+ let docOutput = await splitter.splitDocuments([
|
|
|
+ new Document({ pageContent: this.content }),
|
|
|
+ ]);
|
|
|
+ console.log(docOutput)
|
|
|
+ this.docList = docOutput
|
|
|
+ return this.docList
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 文本向量提取
|
|
|
+ * @see
|
|
|
+ * https://js.langchain.com/docs/integrations/text_embedding/tensorflow/
|
|
|
+ * @returns
|
|
|
+ */
|
|
|
+ // TensorFlow embedding vector(512) NOT NULL -- NOTE: 512 for Tensorflow
|
|
|
+ // OpenAI embedding vector(1536) NOT NULL -- NOTE: 1536 for ChatGPT
|
|
|
+ async embedings(){
|
|
|
+ if(!this.docList?.length){return}
|
|
|
+ const embeddings = new TensorFlowEmbeddings();
|
|
|
+ let documentRes = await embeddings.embedDocuments(this.docList?.map(item=>item.pageContent));
|
|
|
+ console.log(documentRes);
|
|
|
+
|
|
|
+ // 向量持久化
|
|
|
+ documentRes.forEach(async (vector512:any,index)=>{
|
|
|
+ /**
|
|
|
+ * metadata
|
|
|
+ * pageContent
|
|
|
+ */
|
|
|
+ let document = this.docList[index]
|
|
|
+ this.docList[index].vector512 = vector512
|
|
|
+ let hash = await arrayBufferToHASH(stringToArrayBuffer(document?.pageContent))
|
|
|
+ let query = new CloudQuery("Document");
|
|
|
+ query.equalTo("hash",hash);
|
|
|
+ let docObj = await query.first()
|
|
|
+ if(!docObj?.id){
|
|
|
+ docObj = new CloudObject("Document");
|
|
|
+ }
|
|
|
+ docObj.set({
|
|
|
+ metadata:document?.metadata,
|
|
|
+ pageContent:document?.pageContent,
|
|
|
+ vector512:vector512,
|
|
|
+ hash:hash,
|
|
|
+ story:this.story?.toPointer(),
|
|
|
+ })
|
|
|
+ docObj.save();
|
|
|
+ })
|
|
|
+ return documentRes;
|
|
|
+ }
|
|
|
+ async destoryAllDocument(){
|
|
|
+ if(this.story?.id){
|
|
|
+ let query = new CloudQuery("Document");
|
|
|
+ query.equalTo("story",this.story?.id);
|
|
|
+ let docList = await query.find();
|
|
|
+ docList.forEach(doc=>{
|
|
|
+ doc.destroy();
|
|
|
+ })
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+export async function fetchFileBuffer(url: string): Promise<Buffer> {
|
|
|
+ const response = await fetch(url);
|
|
|
+
|
|
|
+ if (!response.ok) {
|
|
|
+ throw new Error(`Failed to fetch file: ${response.status} ${response.statusText}`);
|
|
|
+ }
|
|
|
+
|
|
|
+ const arrayBuffer = await response.arrayBuffer();
|
|
|
+ return Buffer.from(arrayBuffer);
|
|
|
+}
|
|
|
+
|
|
|
+async function setBackend(){
|
|
|
+
|
|
|
+ let backend
|
|
|
+ let WebGPU = (navigator as any).gpu
|
|
|
+ if (WebGPU) {
|
|
|
+ // WebGPU is supported
|
|
|
+ // console.log(WebGPU)
|
|
|
+ backend = "webgpu"
|
|
|
+ } else {
|
|
|
+ // WebGPU is not supported
|
|
|
+ }
|
|
|
+ let glcanvas = document.createElement('canvas');
|
|
|
+ let WebGL = glcanvas.getContext('webgl') || glcanvas.getContext('experimental-webgl');
|
|
|
+ if (WebGL) {
|
|
|
+ // console.log(WebGL)
|
|
|
+ // WebGL is supported
|
|
|
+ if(!backend) backend = "webgl"
|
|
|
+ } else {
|
|
|
+ // WebGL is not supported
|
|
|
+ }
|
|
|
+
|
|
|
+ if (typeof WebAssembly === 'object' && typeof WebAssembly.instantiate === 'function') {
|
|
|
+ // WebAssembly is supported
|
|
|
+ // console.log(WebAssembly)
|
|
|
+ if(!backend) backend = "wasm"
|
|
|
+ } else {
|
|
|
+ // WebAssembly is not supported
|
|
|
+ }
|
|
|
+
|
|
|
+ backend&&await tf.setBackend(backend);
|
|
|
+ await tf.ready();
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ export async function arrayBufferToHASH(arrayBuffer:any) {
|
|
|
+ // 使用 SubtleCrypto API 计算哈希
|
|
|
+ const hashBuffer = await crypto.subtle.digest('SHA-256', arrayBuffer); // 使用 SHA-256 代替 MD5
|
|
|
+ const hashArray = Array.from(new Uint8Array(hashBuffer)); // 将缓冲区转换为字节数组
|
|
|
+ const hashHex = hashArray.map(b => ('00' + b.toString(16)).slice(-2)).join(''); // 转换为十六进制字符串
|
|
|
+ return hashHex;
|
|
|
+}
|
|
|
+export function stringToArrayBuffer(str:string) {
|
|
|
+ // 创建一个与字符串长度相同的Uint8Array
|
|
|
+ const encoder = new TextEncoder();
|
|
|
+ return encoder.encode(str).buffer;
|
|
|
+}
|
|
|
+export async function EmbedQuery(str:any):Promise<Array<number>>{
|
|
|
+ const embeddings = new TensorFlowEmbeddings();
|
|
|
+ let documentRes = await embeddings.embedQuery(str);
|
|
|
+ return documentRes
|
|
|
+}
|
|
|
+
|
|
|
+/** 向量余弦相似度计算 */
|
|
|
+export function RetriveAllDocument(vector1: Array<number>, docList: Array<any>): Array<any> {
|
|
|
+ docList.forEach(doc => {
|
|
|
+ const vector512 = doc.vector512;
|
|
|
+ doc.similarity = cosineSimilarity(vector1, vector512); // 计算余弦相似度并存储
|
|
|
+ });
|
|
|
+
|
|
|
+ // 按照相似度排序，降序排列
|
|
|
+ docList.sort((a, b) => b.similarity - a.similarity);
|
|
|
+
|
|
|
+ return docList; // 返回排序后的docList
|
|
|
+}
|
|
|
+function dotProduct(vectorA: number[], vectorB: number[]): number {
|
|
|
+ return vectorA.reduce((sum, value, index) => sum + value * vectorB[index], 0);
|
|
|
+}
|
|
|
+
|
|
|
+function magnitude(vector: number[]): number {
|
|
|
+ return Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0));
|
|
|
+}
|
|
|
+
|
|
|
+function cosineSimilarity(vectorA: number[], vectorB: number[]): number {
|
|
|
+ const dotProd = dotProduct(vectorA, vectorB);
|
|
|
+ const magnitudeA = magnitude(vectorA);
|
|
|
+ const magnitudeB = magnitude(vectorB);
|
|
|
+
|
|
|
+ if (magnitudeA === 0 || magnitudeB === 0) {
|
|
|
+ throw new Error("One or both vectors are zero vectors, cannot compute cosine similarity.");
|
|
|
+ }
|
|
|
+
|
|
|
+ return dotProd / (magnitudeA * magnitudeB);
|
|
|
+}
|
lzy