RAG 实战教程：从零搭建一个 AI 知识库问答系统

pip install langchain langchain-community langchain-openai chromadb tiktoken openai

import os
os.environ["OPENAI_API_KEY"] = "your-key-here"

from langchain_community.document_loaders import DirectoryLoader

loader = DirectoryLoader(
    "./knowledge_base",
    glob="**/*.md",
    show_progress=True
)
documents = loader.load()

from langchain.text_splitter import RecursiveCharacterTextSplitter

splitter = RecursiveCharacterTextSplitter(
    chunk_size=500,      # 每块500字符
    chunk_overlap=50,    # 重叠50字符
    length_function=len,
)
chunks = splitter.split_documents(documents)
print(f"切分了 {len(chunks)} 个文档块")

from langchain_openai import OpenAIEmbeddings
from langchain_chroma import Chroma

embeddings = OpenAIEmbeddings(model="text-embedding-3-small")

vectorstore = Chroma.from_documents(
    documents=chunks,
    embedding=embeddings,
    persist_directory="./chroma_db"
)

retriever = vectorstore.as_retriever(
    search_type="similarity",       # 相似度搜索
    search_kwargs={"k": 3}          # 返回最相关的3个片段
)

retriever = vectorstore.as_retriever(
    search_type="similarity_score_threshold",
    search_kwargs={"k": 5, "score_threshold": 0.7}
)

from langchain_openai import ChatOpenAI
from langchain.chains import create_retrieval_chain
from langchain.chains.combine_documents import create_stuff_documents_chain
from langchain_core.prompts import ChatPromptTemplate

# 定义系统提示词
system_prompt = (
    "你是一个智能问答助手。请根据以下参考资料回答问题。"
    "如果资料中没有相关信息，请诚实回答'我不知道'，不要编造答案。"
    "\n\n{context}"
)

prompt = ChatPromptTemplate.from_messages([
    ("system", system_prompt),
    ("human", "{input}"),
])

# 创建文档组合链
question_answer_chain = create_stuff_documents_chain(
    ChatOpenAI(model="gpt-4o", temperature=0),
    prompt
)

# 创建检索链
rag_chain = create_retrieval_chain(retriever, question_answer_chain)

# 执行问答
response = rag_chain.invoke({"input": "你们公司的请假政策是什么？"})
print(response["answer"])

import os
from langchain_community.document_loaders import DirectoryLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_openai import OpenAIEmbeddings, ChatOpenAI
from langchain_chroma import Chroma
from langchain.chains import create_retrieval_chain
from langchain.chains.combine_documents import create_stuff_documents_chain
from langchain_core.prompts import ChatPromptTemplate

# 1. 加载文档
loader = DirectoryLoader("./knowledge_base", glob="**/*.md")
documents = loader.load()

# 2. 切分文档
splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
chunks = splitter.split_documents(documents)

# 3. 构建向量库（首次运行）
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
vectorstore = Chroma.from_documents(chunks, embeddings, persist_directory="./chroma_db")

# 4. 或者加载已有向量库
# vectorstore = Chroma(persist_directory="./chroma_db", embedding_function=embeddings)

# 5. 创建检索器
retriever = vectorstore.as_retriever(search_type="similarity", search_kwargs={"k": 3})

# 6. 构建问答链
system_prompt = (
    "你是一个智能问答助手。请根据以下参考资料回答问题。"
    "如果资料中没有相关信息，请回答'抱歉，我没有找到相关内容'。"
    "\n\n{context}"
)
prompt = ChatPromptTemplate.from_messages([
    ("system", system_prompt),
    ("human", "{input}"),
])
qa_chain = create_stuff_documents_chain(ChatOpenAI(model="gpt-4o", temperature=0), prompt)
rag_chain = create_retrieval_chain(retriever, qa_chain)

# 7. 开始问答
while True:
    query = input("你问: ")
    if query == "退出":
        break
    result = rag_chain.invoke({"input": query})
    print(f"回答: {result['answer']}")

retriever = vectorstore.as_retriever(
    search_type="mmr",           # 最大边际相关性
    search_kwargs={"k": 5, "fetch_k": 20}
)

from langchain_community.document_loaders import UnstructuredPDFLoader
loader = UnstructuredPDFLoader("document.pdf")

import redis
r = redis.Redis(host='localhost', port=6379, db=0)

cached = r.get(f"query:{hash(query)}")
if cached:
    return cached
# 否则走 RAG 流程...

pip install ragas