229 lines
9.1 KiB
Plaintext
229 lines
9.1 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "f99b8a44",
|
||
"metadata": {},
|
||
"source": [
|
||
"# L6: 检查结果\n",
|
||
"比较简单轻松的一节"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"id": "5daec1c7",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import os\n",
|
||
"import openai\n",
|
||
"\n",
|
||
"# from dotenv import load_dotenv, find_dotenv\n",
|
||
"# _ = load_dotenv(find_dotenv()) # 读取本地的.env环境文件\n",
|
||
"\n",
|
||
"openai.api_key = 'sk-xxxxxxxxxxxx' #更换成你自己的key"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"id": "9c40b32d",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def get_completion_from_messages(messages, model=\"gpt-3.5-turbo\", temperature=0, max_tokens=500):\n",
|
||
" response = openai.ChatCompletion.create(\n",
|
||
" model=model,\n",
|
||
" messages=messages,\n",
|
||
" temperature=temperature, \n",
|
||
" max_tokens=max_tokens, \n",
|
||
" )\n",
|
||
" return response.choices[0].message[\"content\"]"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "59f69c2e",
|
||
"metadata": {},
|
||
"source": [
|
||
"### 检查输出是否有潜在的有害内容\n",
|
||
"重要的就是一个moderation"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 4,
|
||
"id": "943f5396",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"{\n",
|
||
" \"categories\": {\n",
|
||
" \"hate\": false,\n",
|
||
" \"hate/threatening\": false,\n",
|
||
" \"self-harm\": false,\n",
|
||
" \"sexual\": false,\n",
|
||
" \"sexual/minors\": false,\n",
|
||
" \"violence\": false,\n",
|
||
" \"violence/graphic\": false\n",
|
||
" },\n",
|
||
" \"category_scores\": {\n",
|
||
" \"hate\": 2.6680607e-06,\n",
|
||
" \"hate/threatening\": 1.2194433e-08,\n",
|
||
" \"self-harm\": 8.294434e-07,\n",
|
||
" \"sexual\": 3.41087e-05,\n",
|
||
" \"sexual/minors\": 1.5462567e-07,\n",
|
||
" \"violence\": 6.3285606e-06,\n",
|
||
" \"violence/graphic\": 2.9102332e-06\n",
|
||
" },\n",
|
||
" \"flagged\": false\n",
|
||
"}\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"final_response_to_customer = f\"\"\"\n",
|
||
"SmartX ProPhone有一个6.1英寸的显示屏，128GB存储、1200万像素的双摄像头，以及5G。FotoSnap单反相机有一个2420万像素的传感器，1080p视频，3英寸LCD和 \n",
|
||
"可更换的镜头。我们有各种电视，包括CineView 4K电视，55英寸显示屏，4K分辨率、HDR，以及智能电视功能。我们也有SoundMax家庭影院系统，具有5.1声道，1000W输出，无线 \n",
|
||
"重低音扬声器和蓝牙。关于这些产品或我们提供的任何其他产品您是否有任何具体问题？\n",
|
||
"\"\"\"\n",
|
||
"# Moderation是OpenAI的内容审核函数，用于检测这段内容的危害含量\n",
|
||
"\n",
|
||
"response = openai.Moderation.create(\n",
|
||
" input=final_response_to_customer\n",
|
||
")\n",
|
||
"moderation_output = response[\"results\"][0]\n",
|
||
"print(moderation_output)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "f57f8dad",
|
||
"metadata": {},
|
||
"source": [
|
||
"### 检查输出结果是否与提供的产品信息相符合"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 7,
|
||
"id": "552e3d8c",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Y\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# 这是一段电子产品相关的信息\n",
|
||
"system_message = f\"\"\"\n",
|
||
"You are an assistant that evaluates whether \\\n",
|
||
"customer service agent responses sufficiently \\\n",
|
||
"answer customer questions, and also validates that \\\n",
|
||
"all the facts the assistant cites from the product \\\n",
|
||
"information are correct.\n",
|
||
"The product information and user and customer \\\n",
|
||
"service agent messages will be delimited by \\\n",
|
||
"3 backticks, i.e. ```.\n",
|
||
"Respond with a Y or N character, with no punctuation:\n",
|
||
"Y - if the output sufficiently answers the question \\\n",
|
||
"AND the response correctly uses product information\n",
|
||
"N - otherwise\n",
|
||
"\n",
|
||
"Output a single letter only.\n",
|
||
"\"\"\"\n",
|
||
"\n",
|
||
"#这是顾客的提问\n",
|
||
"customer_message = f\"\"\"\n",
|
||
"tell me about the smartx pro phone and \\\n",
|
||
"the fotosnap camera, the dslr one. \\\n",
|
||
"Also tell me about your tvs\"\"\"\n",
|
||
"product_information = \"\"\"{ \"name\": \"SmartX ProPhone\", \"category\": \"Smartphones and Accessories\", \"brand\": \"SmartX\", \"model_number\": \"SX-PP10\", \"warranty\": \"1 year\", \"rating\": 4.6, \"features\": [ \"6.1-inch display\", \"128GB storage\", \"12MP dual camera\", \"5G\" ], \"description\": \"A powerful smartphone with advanced camera features.\", \"price\": 899.99 } { \"name\": \"FotoSnap DSLR Camera\", \"category\": \"Cameras and Camcorders\", \"brand\": \"FotoSnap\", \"model_number\": \"FS-DSLR200\", \"warranty\": \"1 year\", \"rating\": 4.7, \"features\": [ \"24.2MP sensor\", \"1080p video\", \"3-inch LCD\", \"Interchangeable lenses\" ], \"description\": \"Capture stunning photos and videos with this versatile DSLR camera.\", \"price\": 599.99 } { \"name\": \"CineView 4K TV\", \"category\": \"Televisions and Home Theater Systems\", \"brand\": \"CineView\", \"model_number\": \"CV-4K55\", \"warranty\": \"2 years\", \"rating\": 4.8, \"features\": [ \"55-inch display\", \"4K resolution\", \"HDR\", \"Smart TV\" ], \"description\": \"A stunning 4K TV with vibrant colors and smart features.\", \"price\": 599.99 } { \"name\": \"SoundMax Home Theater\", \"category\": \"Televisions and Home Theater Systems\", \"brand\": \"SoundMax\", \"model_number\": \"SM-HT100\", \"warranty\": \"1 year\", \"rating\": 4.4, \"features\": [ \"5.1 channel\", \"1000W output\", \"Wireless subwoofer\", \"Bluetooth\" ], \"description\": \"A powerful home theater system for an immersive audio experience.\", \"price\": 399.99 } { \"name\": \"CineView 8K TV\", \"category\": \"Televisions and Home Theater Systems\", \"brand\": \"CineView\", \"model_number\": \"CV-8K65\", \"warranty\": \"2 years\", \"rating\": 4.9, \"features\": [ \"65-inch display\", \"8K resolution\", \"HDR\", \"Smart TV\" ], \"description\": \"Experience the future of television with this stunning 8K TV.\", \"price\": 2999.99 } { \"name\": \"SoundMax Soundbar\", \"category\": \"Televisions and Home Theater Systems\", \"brand\": \"SoundMax\", \"model_number\": \"SM-SB50\", \"warranty\": \"1 year\", \"rating\": 4.3, \"features\": [ \"2.1 channel\", \"300W output\", \"Wireless subwoofer\", \"Bluetooth\" ], \"description\": \"Upgrade your TV's audio with this sleek and powerful soundbar.\", \"price\": 199.99 } { \"name\": \"CineView OLED TV\", \"category\": \"Televisions and Home Theater Systems\", \"brand\": \"CineView\", \"model_number\": \"CV-OLED55\", \"warranty\": \"2 years\", \"rating\": 4.7, \"features\": [ \"55-inch display\", \"4K resolution\", \"HDR\", \"Smart TV\" ], \"description\": \"Experience true blacks and vibrant colors with this OLED TV.\", \"price\": 1499.99 }\"\"\"\n",
|
||
"\n",
|
||
"q_a_pair = f\"\"\"\n",
|
||
"Customer message: ```{customer_message}```\n",
|
||
"Product information: ```{product_information}```\n",
|
||
"Agent response: ```{final_response_to_customer}```\n",
|
||
"\n",
|
||
"Does the response use the retrieved information correctly?\n",
|
||
"Does the response sufficiently answer the question?\n",
|
||
"\n",
|
||
"Output Y or N\n",
|
||
"\"\"\"\n",
|
||
"#判断相关性\n",
|
||
"messages = [\n",
|
||
" {'role': 'system', 'content': system_message},\n",
|
||
" {'role': 'user', 'content': q_a_pair}\n",
|
||
"]\n",
|
||
"\n",
|
||
"response = get_completion_from_messages(messages, max_tokens=1)\n",
|
||
"print(response)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 6,
|
||
"id": "afb1b82f",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"N\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"another_response = \"life is like a box of chocolates\"\n",
|
||
"q_a_pair = f\"\"\"\n",
|
||
"Customer message: ```{customer_message}```\n",
|
||
"Product information: ```{product_information}```\n",
|
||
"Agent response: ```{another_response}```\n",
|
||
"\n",
|
||
"Does the response use the retrieved information correctly?\n",
|
||
"Does the response sufficiently answer the question?\n",
|
||
"\n",
|
||
"Output Y or N\n",
|
||
"\"\"\"\n",
|
||
"messages = [\n",
|
||
" {'role': 'system', 'content': system_message},\n",
|
||
" {'role': 'user', 'content': q_a_pair}\n",
|
||
"]\n",
|
||
"\n",
|
||
"response = get_completion_from_messages(messages)\n",
|
||
"print(response)"
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3 (ipykernel)",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.10.9"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|