Rag操作-Ragas评估

一、Ragas评估指标List of available metrics - Ragas

AnswerRelevancy (答案相关性)

二、评估代码实现

# 一、文档处理操作 # 二、构建ragas评估数据格式 # 三、ragas评估代码

import asyncio import pandas as pd from langchain_community.document_loaders import TextLoader from langchain_openai import ChatOpenAI from openai import OpenAI # 一、文档处理操作 # 转为document对象 loader = TextLoader("./text.txt", encoding="utf-8") document = loader.load() print(document) # 分割 from langchain_text_splitters import RecursiveCharacterTextSplitter text_splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=50) documents = text_splitter.split_documents(document) print(documents) # 向量存储 import os from langchain_community.embeddings import DashScopeEmbeddings from langchain_core.vectorstores import InMemoryVectorStore # 1. 初始化通义千问的 embedding 模型 llm_embeddings = DashScopeEmbeddings( model="text-embedding-v4", # 也可以 text-embedding-v1/v2/v3 dashscope_api_key=os.getenv("ali_api_key") ) # 2. 创建向量存储 vector_store = InMemoryVectorStore(llm_embeddings) # 3. 灌文档（你的 langchain_documents 列表） # _ = vector_store.from_documents(documents,llm_embeddings) # vector_store = InMemoryVectorStore(llm_embeddings) vector_store.add_documents(documents) # 检索器检索 retriever = vector_store.as_retriever(search_kwargs={"k": 2}) from langchain_core.prompts import ChatPromptTemplate from langchain_core.output_parsers import StrOutputParser api_key = os.getenv("ali_api_key") model = "qwen-plus" base_url = "https://dashscope.aliyuncs.com/compatible-mode/v1" # langchain调用访问大模型 llm_model = ChatOpenAI(api_key=api_key, base_url=base_url, model=model) template = """Answer the question based only on the following context: {context} Question: {query} """ prompt = ChatPromptTemplate.from_template(template) qa_chain = prompt | llm_model | StrOutputParser() res = qa_chain.invoke({"context": retriever.invoke("Who is the CEO of OpenAI?"), "query": "Who is the CEO of OpenAI?"}) print(res) # 二、构建ragas评估数据格式 #---------- # 问题 # 检索器内容 # 模型答案 # 实际内容 # ------------ sample_queries = [ "Which CEO is widely recognized for democratizing AI education through platforms like Coursera?", "Who is Sam Altman?", "Who is Demis Hassabis and how did he gained prominence?", ] expected_responses = [ "Andrew Ng is the CEO of Landing AI and is widely recognized for democratizing AI education through platforms like Coursera.", "Sam Altman is the CEO of OpenAI and has played a key role in advancing AI research and development. He strongly advocates for creating safe and beneficial AI technologies.", "Demis Hassabis is the CEO of DeepMind and is celebrated for his innovative approach to artificial intelligence. He gained prominence for developing systems like AlphaGo that can master complex games.", ] from ragas import EvaluationDataset dataset = [] for query, reference in zip(sample_queries, expected_responses): relevant_docs = retriever.invoke(query) response = qa_chain.invoke({"context": retriever.invoke(query), "query": query}) # 键必须是这四个字段 dataset.append( { "user_input": query, "retrieved_contexts": [rdoc.page_content for rdoc in relevant_docs], "response": response, "reference": reference, } ) print(dataset) # 列表里面是字典 # [ # {'user_input': 'Which CEO is widely recognized for democratizing AI education through platforms like Coursera?', # 'retrieved_contexts': ['He is also widely recognized for democratizing AI education through platforms like Coursera.",'], # 'response': 'The CEO of OpenAI is Sam Altman.', # 'reference': 'Andrew Ng is the CEO of Landing AI and is widely recognized for democratizing AI education through platforms like Coursera.'}, # ] # 字典列表,封装成 Ragas 专用的 EvaluationDataset对象 evaluation_dataset = EvaluationDataset.from_list(dataset) print(evaluation_dataset) # 三、ragas评估过程 # ragas=0.2.0版本 from ragas import evaluate from ragas.metrics import AnswerCorrectness,AnswerRelevancy # 转为ragas可使用的模型实例 from ragas.llms import LangchainLLMWrapper from ragas.embeddings import LangchainEmbeddingsWrapper # 转换为ragas可识别的模型 embeddings = LangchainEmbeddingsWrapper(llm_embeddings) evaluator_llm = LangchainLLMWrapper(llm_model) # 评估 result = evaluate( dataset=evaluation_dataset, metrics=[ AnswerRelevancy(llm=evaluator_llm,embeddings=embeddings), AnswerCorrectness(llm=evaluator_llm,embeddings=embeddings,weights=[0.5,0.5]) ], llm=evaluator_llm, ) df = result.to_pandas() print(df) print(result) # 输出模板 import pandas as pd from openpyxl import Workbook # # 1. 显示所有列（不省略中间的列） pd.set_option('display.max_columns', None) # # 2. 显示每一列的完整内容（不省略长文字，比如 user_input） pd.set_option('display.max_colwidth', None) # # 3. 设置显示宽度为 None，防止终端换行导致看错乱（可选） pd.set_option('display.width', None)