1、base_llm.py代码#!/usr/bin/env python # -*- coding: UTF-8 -*- from dotenv import load_dotenv from langchain_huggingface import HuggingFaceEmbeddings from langchain_openai import ChatOpenAI import os load_dotenv() llm ChatOpenAI(api_keyos.getenv(DASHSCOPE_API_KEY), base_urlos.getenv(DASHSCOPE_BASE_URL), model_nameqwen3.7-plus) # 本地embedding模型地址 # embedding_model_path rD:\LLM\Local_model\BAAI\bge-large-zh-v1___5 embedding_model_path rD:\LLM\Local_model\maidalun\bce-embedding-base_v1 # 初始化嵌入模型用于文本向量化 embeddings_model HuggingFaceEmbeddings( model_nameembedding_model_path )2、摘要索引代码#!/usr/bin/env python # -*- coding: UTF-8 -*- from base_llm import llm, embeddings_model from langchain_core.stores import InMemoryByteStore from langchain_chroma import Chroma from langchain_community.document_loaders import UnstructuredWordDocumentLoader, WebBaseLoader from langchain_text_splitters import RecursiveCharacterTextSplitter # MultiVectorRetriever 多用检索器 from langchain_classic.retrievers import MultiVectorRetriever import uuid from langchain_core.documents import Document from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import RunnableMap url https://news.pku.edu.cn/mtbdnew/15ac0b3e79244efa88b03a570cbcbcaa.htm # 初始化文档加载器列表加载多个文本文件 loaders [ UnstructuredWordDocumentLoader(人事管理流程.docx), WebBaseLoader(url) ] docs [] for loader in loaders: docs.extend(loader.load()) # 分割器 text_split RecursiveCharacterTextSplitter(chunk_size1024, chunk_overlap100) docs text_split.split_documents(docs) # 创建摘要生成链 chain ( {doc: lambda x: x.page_content} | ChatPromptTemplate.from_template(总结下面的文档:\n\n{doc}) | llm | StrOutputParser() ) summ chain.batch(docs, {max_concurrency: 5}) # 存摘要之后转换成emb的内容 vector Chroma(collection_namesumm, embedding_functionembeddings_model) # 存原文档 store InMemoryByteStore() id_key doc_id # 创建多用检索器 retriever MultiVectorRetriever( vectorstorevector, byte_storestore, id_keyid_key ) # docs 长度100 生成100个随机数 doc_ids [str(uuid.uuid4()) for i in docs] # 创建摘要文档列表包含元数据 metadata可以用来关联原始文档和摘要文档会自动找到匹配关系 summary_docs [ Document(page_contents, metadata{id_key: doc_ids[i]}) for i, s in enumerate(summ) ] # 把压缩之后的内容添加到向量数据库 retriever.vectorstore.add_documents(summary_docs) # 源文档加载到内容 retriever.docstore.mset(list(zip(doc_ids, docs))) # prompt ChatPromptTemplate.from_template(根据下面的文档回答问题:\n\n{doc}\n\n问题: {question}) # # 生成问题回答链 # chain RunnableMap({ # doc: lambda x: retriever.invoke(x[question]), # question: lambda x: x[question] # }) | prompt | llm | StrOutputParser() # # # # 生成问题回答 # query 病假的请假流程? # answer chain.invoke({question: query}) # print(-------------回答--------------) # print(answer) # retrieved_docs retriever.invoke(query) # print(-------------检索到的文档--------------) # print(retrieved_docs) sub_docs retriever.vectorstore.similarity_search(病假的请假流程?) print(sub_docs) print(-------------检索到的文档--------------) print(sub_docs[0]) summ_id sub_docs[0].metadata[id_key] orig_doc retriever.docstore.mget([summ_id]) print(-------------原始文档--------------) print(orig_doc)3、执行报错内容如下C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Scripts\python.exe C:\Users\lenovo\PycharmProjects\PythonProject\rag\day07\01-摘要索引.py C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\langchain_core\utils\pydantic.py:41: UserWarning: Core Pydantic V1 functionality isnt compatible with Python 3.14 or greater. from pydantic.v1 import BaseModel as BaseModelV1 Loading weights: 100%|██████████| 199/199 [00:0000:00, 66810.73it/s] C:\Users\lenovo\PycharmProjects\PythonProject\rag\day07\01-摘要索引.py:8: DeprecationWarning: langchain-community is being sunset and is no longer actively maintained. See https://github.com/langchain-ai/langchain-community/issues/674 for details and migration guidance toward standalone integration packages. from langchain_community.document_loaders import UnstructuredWordDocumentLoader, WebBaseLoader USER_AGENT environment variable not set, consider setting it to identify your requests. [nltk_data] Error loading averaged_perceptron_tagger_eng: [WinError [nltk_data] 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应连接尝试失败。 [nltk_data] Error loading punkt_tab: [WinError 10060] [nltk_data] 由于连接方在一段时间后没有正确答复或连接的主机没有反应连接尝试失败。 Traceback (most recent call last): File C:\Users\lenovo\PycharmProjects\PythonProject\rag\day07\01-摘要索引.py, line 30, in module docs.extend(loader.load()) ~~~~~~~~~~~^^ File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\langchain_core\document_loaders\base.py, line 43, in load return list(self.lazy_load()) File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\langchain_community\document_loaders\unstructured.py, line 107, in lazy_load elements self._get_elements() File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\langchain_community\document_loaders\word_document.py, line 139, in _get_elements return partition_docx(filenameself.file_path, **self.unstructured_kwargs) File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\unstructured\partition\common\metadata.py, line 161, in wrapper elements func(*args, **kwargs) File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\unstructured\chunking\dispatch.py, line 74, in wrapper elements func(*args, **kwargs) File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\unstructured\partition\docx.py, line 182, in partition_docx return list(elements) File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\unstructured\partition\docx.py, line 413, in _iter_document_elements yield from self._iter_paragraph_elements(block_item) File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\unstructured\partition\docx.py, line 637, in _iter_paragraph_elements yield from self._classify_paragraph_to_element(item) File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\unstructured\partition\docx.py, line 474, in _classify_paragraph_to_element TextSubCls self._parse_paragraph_text_for_element_type(paragraph) File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\unstructured\partition\docx.py, line 948, in _parse_paragraph_text_for_element_type if is_possible_narrative_text(text): ~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\unstructured\partition\text_type.py, line 74, in is_possible_narrative_text if exceeds_cap_ratio(text, thresholdcap_threshold): ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\unstructured\partition\text_type.py, line 282, in exceeds_cap_ratio if sentence_count(text, 3) 1: ~~~~~~~~~~~~~~^^^^^^^^^ File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\unstructured\partition\text_type.py, line 219, in sentence_count sentences sent_tokenize(text) File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\unstructured\nlp\tokenize.py, line 55, in sent_tokenize return list(_tokenize_for_cache(text)) ~~~~~~~~~~~~~~~~~~~^^^^^^ File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\unstructured\nlp\tokenize.py, line 80, in _tokenize_for_cache return tuple(_sent_tokenize(text)) ~~~~~~~~~~~~~~^^^^^^ File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\nltk\tokenize\__init__.py, line 119, in sent_tokenize tokenizer _get_punkt_tokenizer(language) File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\nltk\tokenize\__init__.py, line 105, in _get_punkt_tokenizer return PunktTokenizer(language) File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\nltk\tokenize\punkt.py, line 1746, in __init__ self.load_lang(lang) ~~~~~~~~~~~~~~^^^^^^ File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\nltk\tokenize\punkt.py, line 1751, in load_lang lang_dir find(ftokenizers/punkt_tab/{lang}/) File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\nltk\data.py, line 693, in find return find(modified_name, paths) File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\nltk\data.py, line 679, in find return ZipFilePathPointer(p, zipentry) File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\nltk\data.py, line 471, in __init__ zipfile OpenOnDemandZipFile(os.path.abspath(zipfile)) File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\nltk\data.py, line 1192, in __init__ ZipFile.__init__(self, filename) ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^ File C:\Users\lenovo\PycharmProjects\PythonProject\.venv\Lib\site-packages\nltk\pathsec.py, line 265, in __init__ super().__init__(file, *args, **kwargs) ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ File C:\Users\lenovo\AppData\Local\Programs\Python\Python314\Lib\zipfile\__init__.py, line 1458, in __init__ self._RealGetContents() ~~~~~~~~~~~~~~~~~~~~~^^ File C:\Users\lenovo\AppData\Local\Programs\Python\Python314\Lib\zipfile\__init__.py, line 1525, in _RealGetContents raise BadZipFile(File is not a zip file) zipfile.BadZipFile: File is not a zip file Process finished with exit code 1