From d24fbc681ca76ede8159e3ba0076bce9881ff48f Mon Sep 17 00:00:00 2001 From: Logan Zou <74288839+nowadays0421@users.noreply.github.com> Date: Fri, 2 Jun 2023 01:06:23 +0800 Subject: [PATCH] Add files via upload --- .../7.Evaluation.ipynb | 515 +++++++++++++ .../9.Evaluation-part2.ipynb | 688 ++++++++++++++++++ 2 files changed, 1203 insertions(+) create mode 100644 content/Building Systems with the ChatGPT API/7.Evaluation.ipynb create mode 100644 content/Building Systems with the ChatGPT API/9.Evaluation-part2.ipynb diff --git a/content/Building Systems with the ChatGPT API/7.Evaluation.ipynb b/content/Building Systems with the ChatGPT API/7.Evaluation.ipynb new file mode 100644 index 0000000..9e85637 --- /dev/null +++ b/content/Building Systems with the ChatGPT API/7.Evaluation.ipynb @@ -0,0 +1,515 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 第七章 搭建一个带评估的端到端问答系统" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "环境配置" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "application/javascript": "(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, js_modules, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n if (js_modules == null) js_modules = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls.length === 0 && js_modules.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n var skip = [];\n if (window.requirejs) {\n window.requirejs.config({'packages': {}, 'paths': {'gridstack': 'https://cdn.jsdelivr.net/npm/gridstack@4.2.5/dist/gridstack-h5', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'gridstack': {'exports': 'GridStack'}}});\n require([\"gridstack\"], function(GridStack) {\n\twindow.GridStack = GridStack\n\ton_load()\n })\n require([\"notyf\"], function() {\n\ton_load()\n })\n root._bokeh_is_loading = css_urls.length + 2;\n } else {\n root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length;\n } if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/0.14.4/dist/bundled/gridstack/gridstack@4.2.5/dist/gridstack-h5.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/0.14.4/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n if (skip.indexOf(url) >= 0) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (var i = 0; i < js_modules.length; i++) {\n var url = js_modules[i];\n if (skip.indexOf(url) >= 0) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n if (!js_urls.length && !js_modules.length) {\n on_load()\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-2.4.3.min.js\", \"https://unpkg.com/@holoviz/panel@0.14.4/dist/panel.min.js\"];\n var js_modules = [];\n var css_urls = [\"https://cdn.holoviz.org/panel/0.14.4/dist/css/alerts.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/card.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/dataframe.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/debugger.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/json.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/loading.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/markdown.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/widgets.css\"];\n var inline_js = [ function(Bokeh) {\n inject_raw_css(\"\\n .bk.pn-loading.arc:before {\\n background-image: url(\\\"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHN0eWxlPSJtYXJnaW46IGF1dG87IGJhY2tncm91bmQ6IG5vbmU7IGRpc3BsYXk6IGJsb2NrOyBzaGFwZS1yZW5kZXJpbmc6IGF1dG87IiB2aWV3Qm94PSIwIDAgMTAwIDEwMCIgcHJlc2VydmVBc3BlY3RSYXRpbz0ieE1pZFlNaWQiPiAgPGNpcmNsZSBjeD0iNTAiIGN5PSI1MCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjYzNjM2MzIiBzdHJva2Utd2lkdGg9IjEwIiByPSIzNSIgc3Ryb2tlLWRhc2hhcnJheT0iMTY0LjkzMzYxNDMxMzQ2NDE1IDU2Ljk3Nzg3MTQzNzgyMTM4Ij4gICAgPGFuaW1hdGVUcmFuc2Zvcm0gYXR0cmlidXRlTmFtZT0idHJhbnNmb3JtIiB0eXBlPSJyb3RhdGUiIHJlcGVhdENvdW50PSJpbmRlZmluaXRlIiBkdXI9IjFzIiB2YWx1ZXM9IjAgNTAgNTA7MzYwIDUwIDUwIiBrZXlUaW1lcz0iMDsxIj48L2FuaW1hdGVUcmFuc2Zvcm0+ICA8L2NpcmNsZT48L3N2Zz4=\\\");\\n background-size: auto calc(min(50%, 400px));\\n }\\n \");\n }, function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n ];\n\n function run_inline_js() {\n if ((root.Bokeh !== undefined) || (force === true)) {\n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, js_modules, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));", + "application/vnd.holoviews_load.v0+json": "" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n function JupyterCommManager() {\n }\n\n JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n comm_manager.register_target(comm_id, function(comm) {\n comm.on_msg(msg_handler);\n });\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n comm.onMsg = msg_handler;\n });\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n console.log(message)\n var content = {data: message.data, comm_id};\n var buffers = []\n for (var buffer of message.buffers || []) {\n buffers.push(new DataView(buffer))\n }\n var metadata = message.metadata || {};\n var msg = {content, buffers, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n })\n }\n }\n\n JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n if (comm_id in window.PyViz.comms) {\n return window.PyViz.comms[comm_id];\n } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n if (msg_handler) {\n comm.on_msg(msg_handler);\n }\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n comm.open();\n if (msg_handler) {\n comm.onMsg = msg_handler;\n }\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n var comm_promise = google.colab.kernel.comms.open(comm_id)\n comm_promise.then((comm) => {\n window.PyViz.comms[comm_id] = comm;\n if (msg_handler) {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n var content = {data: message.data};\n var metadata = message.metadata || {comm_id};\n var msg = {content, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n }\n }) \n var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n return comm_promise.then((comm) => {\n comm.send(data, metadata, buffers, disposeOnDone);\n });\n };\n var comm = {\n send: sendClosure\n };\n }\n window.PyViz.comms[comm_id] = comm;\n return comm;\n }\n window.PyViz.comm_manager = new JupyterCommManager();\n \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n var div = document.createElement(\"div\");\n var script = document.createElement(\"script\");\n node.appendChild(div);\n node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n var output_area = handle.output_area;\n var output = handle.output;\n if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n return\n }\n var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n if (id !== undefined) {\n var nchildren = toinsert.length;\n var html_node = toinsert[nchildren-1].children[0];\n html_node.innerHTML = output.data[HTML_MIME_TYPE];\n var scripts = [];\n var nodelist = html_node.querySelectorAll(\"script\");\n for (var i in nodelist) {\n if (nodelist.hasOwnProperty(i)) {\n scripts.push(nodelist[i])\n }\n }\n\n scripts.forEach( function (oldScript) {\n var newScript = document.createElement(\"script\");\n var attrs = [];\n var nodemap = oldScript.attributes;\n for (var j in nodemap) {\n if (nodemap.hasOwnProperty(j)) {\n attrs.push(nodemap[j])\n }\n }\n attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n oldScript.parentNode.replaceChild(newScript, oldScript);\n });\n if (JS_MIME_TYPE in output.data) {\n toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n }\n output_area._hv_plot_id = id;\n if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n window.PyViz.plot_index[id] = Bokeh.index[id];\n } else {\n window.PyViz.plot_index[id] = null;\n }\n } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n var bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n var script_attrs = bk_div.children[0].attributes;\n for (var i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n var id = handle.cell.output_area._hv_plot_id;\n var server_id = handle.cell.output_area._bokeh_server_id;\n if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n if (server_id !== null) {\n comm.send({event_type: 'server_delete', 'id': server_id});\n return;\n } else if (comm !== null) {\n comm.send({event_type: 'delete', 'id': id});\n }\n delete PyViz.plot_index[id];\n if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n var doc = window.Bokeh.index[id].model.document\n doc.clear();\n const i = window.Bokeh.documents.indexOf(doc);\n if (i > -1) {\n window.Bokeh.documents.splice(i, 1);\n }\n }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n delete PyViz.comms[\"hv-extension-comm\"];\n window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n handle_clear_output(event, {cell: {output_area: handle.output_area}})\n handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n var toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[0]);\n element.append(toinsert);\n return toinsert\n }\n\n events.on('output_added.OutputArea', handle_add_output);\n events.on('output_updated.OutputArea', handle_update_output);\n events.on('clear_output.CodeCell', handle_clear_output);\n events.on('delete.Cell', handle_clear_output);\n events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n safe: true,\n index: 0\n });\n}\n\nif (window.Jupyter !== undefined) {\n try {\n var events = require('base/js/events');\n var OutputArea = require('notebook/js/outputarea').OutputArea;\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n } catch(err) {\n }\n}\n", + "application/vnd.holoviews_load.v0+json": "" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# 配置 OpenAI KEY\n", + "import os\n", + "import openai\n", + "import sys\n", + "sys.path.append('../..')\n", + "# 使用英文 Prompt 的工具包\n", + "import utils_en\n", + "# 使用中文 Prompt 的工具包\n", + "import utils_ch\n", + "\n", + "import panel as pn # GUI\n", + "pn.extension()\n", + "\n", + "from dotenv import load_dotenv, find_dotenv\n", + "_ = load_dotenv(find_dotenv()) # read local .env file\n", + "\n", + "openai.api_key = os.environ['OPENAI_API_KEY']" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# 封装一个访问 OpenAI GPT3.5 的函数\n", + "def get_completion_from_messages(messages, model=\"gpt-3.5-turbo\", temperature=0, max_tokens=500):\n", + " response = openai.ChatCompletion.create(\n", + " model=model,\n", + " messages=messages,\n", + " temperature=temperature, \n", + " max_tokens=max_tokens, \n", + " )\n", + " return response.choices[0].message[\"content\"]" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "一个端到端实现问答的函数" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "第一步：输入通过 Moderation 检查\n", + "第二步：抽取出商品列表\n", + "第三步：查找抽取出的商品信息\n", + "第四步：生成用户回答\n", + "第五步：输出经过 Moderation 检查\n", + "第六步：模型评估该回答\n", + "第七步：模型赞同了该回答.\n", + "The SmartX ProPhone is a powerful smartphone with a 6.1-inch display, 128GB storage, 12MP dual camera, and 5G capabilities. The FotoSnap DSLR Camera is a versatile camera with a 24.2MP sensor, 1080p video, 3-inch LCD, and interchangeable lenses. As for our TVs, we have a range of options including the CineView 4K TV with a 55-inch display, 4K resolution, HDR, and smart TV capabilities, the CineView 8K TV with a 65-inch display, 8K resolution, HDR, and smart TV capabilities, and the CineView OLED TV with a 55-inch display, 4K resolution, HDR, and smart TV capabilities. Do you have any specific questions about these products or would you like me to recommend a product based on your needs?\n" + ] + } + ], + "source": [ + "# 对用户信息进行预处理\n", + "def process_user_message(user_input, all_messages, debug=True):\n", + " # user_input : 用户输入\n", + " # all_messages : 历史信息\n", + " # debug : 是否开启 DEBUG 模式,默认开启\n", + "\n", + " # 分隔符\n", + " delimiter = \"```\"\n", + " \n", + " # 第一步: 使用 OpenAI 的 Moderation API 检查用户输入是否合规或者是一个注入的 Prompt\n", + " response = openai.Moderation.create(input=user_input)\n", + " moderation_output = response[\"results\"][0]\n", + "\n", + " # 经过 Moderation API 检查该输入不合规\n", + " if moderation_output[\"flagged\"]:\n", + " print(\"第一步：输入被 Moderation 拒绝\")\n", + " return \"抱歉，您的请求不合规\"\n", + "\n", + " # 如果开启了 DEBUG 模式，打印实时进度\n", + " if debug: print(\"第一步：输入通过 Moderation 检查\")\n", + " \n", + " # 第二步：抽取出商品和对应的目录，类似于之前课程中的方法，做了一个封装\n", + " category_and_product_response = utils_en.find_category_and_product_only(user_input, utils_en.get_products_and_category())\n", + " #print(category_and_product_response)\n", + " # 将抽取出来的字符串转化为列表\n", + " category_and_product_list = utils_en.read_string_to_list(category_and_product_response)\n", + " #print(category_and_product_list)\n", + "\n", + " if debug: print(\"第二步：抽取出商品列表\")\n", + "\n", + " # 第三步：查找商品对应信息\n", + " product_information = utils_en.generate_output_string(category_and_product_list)\n", + " if debug: print(\"第三步：查找抽取出的商品信息\")\n", + "\n", + " # 第四步：根据信息生成回答\n", + " system_message = f\"\"\"\n", + " You are a customer service assistant for a large electronic store. \\\n", + " Respond in a friendly and helpful tone, with concise answers. \\\n", + " Make sure to ask the user relevant follow-up questions.\n", + " \"\"\"\n", + " # 插入 message\n", + " messages = [\n", + " {'role': 'system', 'content': system_message},\n", + " {'role': 'user', 'content': f\"{delimiter}{user_input}{delimiter}\"},\n", + " {'role': 'assistant', 'content': f\"Relevant product information:\\n{product_information}\"}\n", + " ]\n", + " # 获取 GPT3.5 的回答\n", + " # 通过附加 all_messages 实现多轮对话\n", + " final_response = get_completion_from_messages(all_messages + messages)\n", + " if debug:print(\"第四步：生成用户回答\")\n", + " # 将该轮信息加入到历史信息中\n", + " all_messages = all_messages + messages[1:]\n", + "\n", + " # 第五步：基于 Moderation API 检查输出是否合规\n", + " response = openai.Moderation.create(input=final_response)\n", + " moderation_output = response[\"results\"][0]\n", + "\n", + " # 输出不合规\n", + " if moderation_output[\"flagged\"]:\n", + " if debug: print(\"第五步：输出被 Moderation 拒绝\")\n", + " return \"抱歉，我们不能提供该信息\"\n", + "\n", + " if debug: print(\"第五步：输出经过 Moderation 检查\")\n", + "\n", + " # 第六步：模型检查是否很好地回答了用户问题\n", + " user_message = f\"\"\"\n", + " Customer message: {delimiter}{user_input}{delimiter}\n", + " Agent response: {delimiter}{final_response}{delimiter}\n", + "\n", + " Does the response sufficiently answer the question?\n", + " \"\"\"\n", + " messages = [\n", + " {'role': 'system', 'content': system_message},\n", + " {'role': 'user', 'content': user_message}\n", + " ]\n", + " # 要求模型评估回答\n", + " evaluation_response = get_completion_from_messages(messages)\n", + " if debug: print(\"第六步：模型评估该回答\")\n", + "\n", + " # 第七步：如果评估为 Y，输出回答；如果评估为 N，反馈将由人工修正答案\n", + " if \"Y\" in evaluation_response: # 使用 in 来避免模型可能生成 Yes\n", + " if debug: print(\"第七步：模型赞同了该回答.\")\n", + " return final_response, all_messages\n", + " else:\n", + " if debug: print(\"第七步：模型不赞成该回答.\")\n", + " neg_str = \"很抱歉，我无法提供您所需的信息。我将为您转接到一位人工客服代表以获取进一步帮助。\"\n", + " return neg_str, all_messages\n", + "\n", + "user_input = \"tell me about the smartx pro phone and the fotosnap camera, the dslr one. Also what tell me about your tvs\"\n", + "response,_ = process_user_message(user_input,[])\n", + "print(response)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "第一步：输入通过 Moderation 检查\n", + "第二步：抽取出商品列表\n", + "第三步：查找抽取出的商品信息\n", + "第四步：生成用户回答\n", + "第五步：输出经过 Moderation 检查\n", + "第六步：模型评估该回答\n", + "第七步：模型赞同了该回答.\n", + "关于SmartX ProPhone和FotoSnap相机的信息：\n", + "\n", + "SmartX ProPhone是一款功能强大的智能手机，具有6.1英寸的显示屏，128GB的存储空间，12MP的双摄像头和5G网络。售价为899.99美元。\n", + "\n", + "FotoSnap相机系列包括DSLR相机、无反相机和即时相机。DSLR相机具有24.2MP传感器、1080p视频、3英寸LCD和可更换镜头。无反相机具有20.1MP传感器、4K视频、3英寸触摸屏和可更换镜头。即时相机可以即时打印照片，具有内置闪光灯、自拍镜和电池供电。售价分别为599.99美元、799.99美元和69.99美元。\n", + "\n", + "关于我们的电视：\n", + "\n", + "我们有多种电视可供选择，包括CineView 4K电视、CineView 8K电视和CineView OLED电视。CineView 4K电视具有55英寸的显示屏、4K分辨率、HDR和智能电视功能。CineView 8K电视具有65英寸的显示屏、8K分辨率、HDR和智能电视功能。CineView OLED电视具有55英寸的显示屏、4K分辨率、HDR和智能电视功能。我们还提供SoundMax家庭影院和SoundMax声音栏，以提供更好的音频体验。售价从199.99美元到2999.99美元不等，保修期为1年或2年。\n" + ] + } + ], + "source": [ + "'''\n", + "中文Prompt\n", + "注意：限于模型对中文理解能力较弱，中文Prompt可能会随机出现不成功，可以多次运行；也非常欢迎同学探究更稳定的中文 Prompt\n", + "'''\n", + "# 对用户信息进行预处理\n", + "def process_user_message_ch(user_input, all_messages, debug=True):\n", + " # user_input : 用户输入\n", + " # all_messages : 历史信息\n", + " # debug : 是否开启 DEBUG 模式,默认开启\n", + "\n", + " # 分隔符\n", + " delimiter = \"```\"\n", + " \n", + " # 第一步: 使用 OpenAI 的 Moderation API 检查用户输入是否合规或者是一个注入的 Prompt\n", + " response = openai.Moderation.create(input=user_input)\n", + " moderation_output = response[\"results\"][0]\n", + "\n", + " # 经过 Moderation API 检查该输入不合规\n", + " if moderation_output[\"flagged\"]:\n", + " print(\"第一步：输入被 Moderation 拒绝\")\n", + " return \"抱歉，您的请求不合规\"\n", + "\n", + " # 如果开启了 DEBUG 模式，打印实时进度\n", + " if debug: print(\"第一步：输入通过 Moderation 检查\")\n", + " \n", + " # 第二步：抽取出商品和对应的目录，类似于之前课程中的方法，做了一个封装\n", + " category_and_product_response = utils_ch.find_category_and_product_only(user_input, utils_ch.get_products_and_category())\n", + " #print(category_and_product_response)\n", + " # 将抽取出来的字符串转化为列表\n", + " category_and_product_list = utils_ch.read_string_to_list(category_and_product_response)\n", + " #print(category_and_product_list)\n", + "\n", + " if debug: print(\"第二步：抽取出商品列表\")\n", + "\n", + " # 第三步：查找商品对应信息\n", + " product_information = utils_ch.generate_output_string(category_and_product_list)\n", + " if debug: print(\"第三步：查找抽取出的商品信息\")\n", + "\n", + " # 第四步：根据信息生成回答\n", + " system_message = f\"\"\"\n", + " 您是一家大型电子商店的客户服务助理。\\\n", + " 请以友好和乐于助人的语气回答问题，并提供简洁明了的答案。\\\n", + " 请确保向用户提出相关的后续问题。\n", + " \"\"\"\n", + " # 插入 message\n", + " messages = [\n", + " {'role': 'system', 'content': system_message},\n", + " {'role': 'user', 'content': f\"{delimiter}{user_input}{delimiter}\"},\n", + " {'role': 'assistant', 'content': f\"相关商品信息:\\n{product_information}\"}\n", + " ]\n", + " # 获取 GPT3.5 的回答\n", + " # 通过附加 all_messages 实现多轮对话\n", + " final_response = get_completion_from_messages(all_messages + messages)\n", + " if debug:print(\"第四步：生成用户回答\")\n", + " # 将该轮信息加入到历史信息中\n", + " all_messages = all_messages + messages[1:]\n", + "\n", + " # 第五步：基于 Moderation API 检查输出是否合规\n", + " response = openai.Moderation.create(input=final_response)\n", + " moderation_output = response[\"results\"][0]\n", + "\n", + " # 输出不合规\n", + " if moderation_output[\"flagged\"]:\n", + " if debug: print(\"第五步：输出被 Moderation 拒绝\")\n", + " return \"抱歉，我们不能提供该信息\"\n", + "\n", + " if debug: print(\"第五步：输出经过 Moderation 检查\")\n", + "\n", + " # 第六步：模型检查是否很好地回答了用户问题\n", + " user_message = f\"\"\"\n", + " 用户信息: {delimiter}{user_input}{delimiter}\n", + " 代理回复: {delimiter}{final_response}{delimiter}\n", + "\n", + " 回复是否足够回答问题\n", + " 如果足够，回答 Y\n", + " 如果不足够，回答 N\n", + " 仅回答上述字母即可\n", + " \"\"\"\n", + " # print(final_response)\n", + " messages = [\n", + " {'role': 'system', 'content': system_message},\n", + " {'role': 'user', 'content': user_message}\n", + " ]\n", + " # 要求模型评估回答\n", + " evaluation_response = get_completion_from_messages(messages)\n", + " # print(evaluation_response)\n", + " if debug: print(\"第六步：模型评估该回答\")\n", + "\n", + " # 第七步：如果评估为 Y，输出回答；如果评估为 N，反馈将由人工修正答案\n", + " if \"Y\" in evaluation_response: # 使用 in 来避免模型可能生成 Yes\n", + " if debug: print(\"第七步：模型赞同了该回答.\")\n", + " return final_response, all_messages\n", + " else:\n", + " if debug: print(\"第七步：模型不赞成该回答.\")\n", + " neg_str = \"很抱歉，我无法提供您所需的信息。我将为您转接到一位人工客服代表以获取进一步帮助。\"\n", + " return neg_str, all_messages\n", + "\n", + "user_input = \"请告诉我关于smartx pro phone和the fotosnap camera的信息。另外，请告诉我关于你们的tvs的情况。\"\n", + "response,_ = process_user_message_ch(user_input,[])\n", + "print(response)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "实现一个可视化界面" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "def collect_messages_en(debug=False):\n", + " user_input = inp.value_input\n", + " if debug: print(f\"User Input = {user_input}\")\n", + " if user_input == \"\":\n", + " return\n", + " inp.value = ''\n", + " global context\n", + " # 调用 process_user_message 函数\n", + " #response, context = process_user_message(user_input, context, utils.get_products_and_category(),debug=True)\n", + " response, context = process_user_message(user_input, context, debug=False)\n", + " context.append({'role':'assistant', 'content':f\"{response}\"})\n", + " panels.append(\n", + " pn.Row('User:', pn.pane.Markdown(user_input, width=600)))\n", + " panels.append(\n", + " pn.Row('Assistant:', pn.pane.Markdown(response, width=600, style={'background-color': '#F6F6F6'})))\n", + " \n", + " return pn.Column(*panels)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 调用中文Prompt版本\n", + "def collect_messages_ch(debug=False):\n", + " user_input = inp.value_input\n", + " if debug: print(f\"User Input = {user_input}\")\n", + " if user_input == \"\":\n", + " return\n", + " inp.value = ''\n", + " global context\n", + " # 调用 process_user_message 函数\n", + " #response, context = process_user_message(user_input, context, utils.get_products_and_category(),debug=True)\n", + " response, context = process_user_message_ch(user_input, context, debug=False)\n", + " context.append({'role':'assistant', 'content':f\"{response}\"})\n", + " panels.append(\n", + " pn.Row('User:', pn.pane.Markdown(user_input, width=600)))\n", + " panels.append(\n", + " pn.Row('Assistant:', pn.pane.Markdown(response, width=600, style={'background-color': '#F6F6F6'})))\n", + " \n", + " return pn.Column(*panels)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": {}, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.holoviews_exec.v0+json": "", + "text/html": [ + "
\n", + "
\n", + "
\n", + "" + ], + "text/plain": [ + "Column\n", + " [0] TextInput(placeholder='Enter text here…')\n", + " [1] Row\n", + " [0] Button(name='Service Assistant')\n", + " [2] ParamFunction(function, _pane=Str, height=300, loading_indicator=True)" + ] + }, + "execution_count": 18, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "1002" + } + }, + "output_type": "execute_result" + } + ], + "source": [ + "panels = [] # collect display \n", + "\n", + "# 系统信息\n", + "context = [ {'role':'system', 'content':\"You are Service Assistant\"} ] \n", + "\n", + "inp = pn.widgets.TextInput( placeholder='Enter text here…')\n", + "button_conversation = pn.widgets.Button(name=\"Service Assistant\")\n", + "\n", + "interactive_conversation = pn.bind(collect_messages, button_conversation)\n", + "\n", + "dashboard = pn.Column(\n", + " inp,\n", + " pn.Row(button_conversation),\n", + " pn.panel(interactive_conversation, loading_indicator=True, height=300),\n", + ")\n", + "\n", + "dashboard" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "zyh_gpt", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/content/Building Systems with the ChatGPT API/9.Evaluation-part2.ipynb b/content/Building Systems with the ChatGPT API/9.Evaluation-part2.ipynb new file mode 100644 index 0000000..5443bc2 --- /dev/null +++ b/content/Building Systems with the ChatGPT API/9.Evaluation-part2.ipynb @@ -0,0 +1,688 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 第九章 评估（下）——当不存在一个简单的正确答案时" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "环境配置" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# 导入 OpenAI API\n", + "import os\n", + "import openai\n", + "import sys\n", + "sys.path.append('../..')\n", + "import utils_en\n", + "import utils_ch\n", + "from dotenv import load_dotenv, find_dotenv\n", + "_ = load_dotenv(find_dotenv()) # read local .env file\n", + "\n", + "openai.api_key = os.environ['OPENAI_API_KEY']" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# 封装一个访问 OpenAI GPT3.5 的函数\n", + "def get_completion_from_messages(messages, model=\"gpt-3.5-turbo\", temperature=0, max_tokens=500):\n", + " response = openai.ChatCompletion.create(\n", + " model=model,\n", + " messages=messages,\n", + " temperature=temperature, \n", + " max_tokens=max_tokens, \n", + " )\n", + " return response.choices[0].message[\"content\"]" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "运行问答系统获得回答" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# 用户消息\n", + "customer_msg = f\"\"\"\n", + "tell me about the smartx pro phone and the fotosnap camera, the dslr one.\n", + "Also, what TVs or TV related products do you have?\"\"\"\n", + "\n", + "# 从问题中抽取商品名\n", + "products_by_category = utils_en.get_products_from_query(customer_msg)\n", + "# 将商品名转化为列表\n", + "category_and_product_list = utils_en.read_string_to_list(products_by_category)\n", + "# 查找商品对应的信息\n", + "product_info = utils_en.get_mentioned_product_info(category_and_product_list)\n", + "# 由信息生成回答\n", + "assistant_answer = utils_en.answer_user_msg(user_msg=customer_msg, product_info=product_info)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sure, I'd be happy to help! The SmartX ProPhone is a powerful smartphone with a 6.1-inch display, 128GB storage, 12MP dual camera, and 5G capabilities. The FotoSnap DSLR Camera is a versatile camera with a 24.2MP sensor, 1080p video, 3-inch LCD, and interchangeable lenses. As for TVs, we have a variety of options including the CineView 4K TV with a 55-inch display, 4K resolution, HDR, and smart TV capabilities, the CineView 8K TV with a 65-inch display, 8K resolution, HDR, and smart TV capabilities, and the CineView OLED TV with a 55-inch display, 4K resolution, HDR, and smart TV capabilities. We also have the SoundMax Home Theater system with 5.1 channel, 1000W output, wireless subwoofer, and Bluetooth, and the SoundMax Soundbar with 2.1 channel, 300W output, wireless subwoofer, and Bluetooth. Is there anything else I can help you with?\n" + ] + } + ], + "source": [ + "print(assistant_answer) " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "'''\n", + "中文Prompt\n", + "注意：限于模型对中文理解能力较弱，中文Prompt可能会随机出现不成功，可以多次运行；也非常欢迎同学探究更稳定的中文 Prompt\n", + "'''\n", + "# 用户消息\n", + "customer_msg = f\"\"\"\n", + "告诉我有关 the smartx pro phone 和 the fotosnap camera, the dslr one 的信息。\n", + "另外，你们这有什么 TVs ？\"\"\"\n", + "\n", + "# 从问题中抽取商品名\n", + "products_by_category = utils_ch.get_products_from_query(customer_msg)\n", + "# 将商品名转化为列表\n", + "category_and_product_list = utils_ch.read_string_to_list(products_by_category)\n", + "# 查找商品对应的信息\n", + "product_info = utils_ch.get_mentioned_product_info(category_and_product_list)\n", + "# 由信息生成回答\n", + "assistant_answer = utils_ch.answer_user_msg(user_msg=customer_msg, product_info=product_info)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "关于SmartX ProPhone和FotoSnap DSLR相机的信息：\n", + "\n", + "SmartX ProPhone是一款功能强大的智能手机，具有6.1英寸的显示屏，128GB的存储空间，12MP双摄像头和5G网络。它的价格为899.99美元，保修期为1年。\n", + "\n", + "FotoSnap DSLR相机是一款功能强大的相机，具有24.2MP传感器，1080p视频，3英寸LCD屏幕和可更换镜头。它的价格为599.99美元，保修期为1年。\n", + "\n", + "关于电视的信息：\n", + "\n", + "我们有多种电视可供选择，包括CineView 4K电视，CineView 8K电视和CineView OLED电视。CineView 4K电视具有55英寸的显示屏，4K分辨率，HDR和智能电视功能，价格为599.99美元，保修期为2年。CineView 8K电视具有65英寸的显示屏，8K分辨率，HDR和智能电视功能，价格为2999.99美元，保修期为2年。CineView OLED电视具有55英寸的显示屏，4K分辨率，HDR和智能电视功能，价格为1499.99美元，保修期为2年。您需要哪种类型的电视？\n" + ] + } + ], + "source": [ + "print(assistant_answer) " + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "使用 GPT 评估回答是否正确" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# 问题、上下文\n", + "cust_prod_info = {\n", + " 'customer_msg': customer_msg,\n", + " 'context': product_info\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "# 使用 GPT API 评估生成的回答\n", + "def eval_with_rubric(test_set, assistant_answer):\n", + "\n", + " cust_msg = test_set['customer_msg']\n", + " context = test_set['context']\n", + " completion = assistant_answer\n", + " \n", + " # 要求 GPT 作为一个助手评估回答正确性\n", + " system_message = \"\"\"\\\n", + " You are an assistant that evaluates how well the customer service agent \\\n", + " answers a user question by looking at the context that the customer service \\\n", + " agent is using to generate its response. \n", + " \"\"\"\n", + "\n", + " # 具体指令\n", + " user_message = f\"\"\"\\\n", + "You are evaluating a submitted answer to a question based on the context \\\n", + "that the agent uses to answer the question.\n", + "Here is the data:\n", + " [BEGIN DATA]\n", + " ************\n", + " [Question]: {cust_msg}\n", + " ************\n", + " [Context]: {context}\n", + " ************\n", + " [Submission]: {completion}\n", + " ************\n", + " [END DATA]\n", + "\n", + "Compare the factual content of the submitted answer with the context. \\\n", + "Ignore any differences in style, grammar, or punctuation.\n", + "Answer the following questions:\n", + " - Is the Assistant response based only on the context provided? (Y or N)\n", + " - Does the answer include information that is not provided in the context? (Y or N)\n", + " - Is there any disagreement between the response and the context? (Y or N)\n", + " - Count how many questions the user asked. (output a number)\n", + " - For each question that the user asked, is there a corresponding answer to it?\n", + " Question 1: (Y or N)\n", + " Question 2: (Y or N)\n", + " ...\n", + " Question N: (Y or N)\n", + " - Of the number of questions asked, how many of these questions were addressed by the answer? (output a number)\n", + "\"\"\"\n", + "\n", + " messages = [\n", + " {'role': 'system', 'content': system_message},\n", + " {'role': 'user', 'content': user_message}\n", + " ]\n", + "\n", + " response = get_completion_from_messages(messages)\n", + " return response" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "- Is the Assistant response based only on the context provided? (Y or N)\n", + "Y\n", + "- Does the answer include information that is not provided in the context? (Y or N)\n", + "N\n", + "- Is there any disagreement between the response and the context? (Y or N)\n", + "N\n", + "- Count how many questions the user asked. (output a number)\n", + "1\n", + "- For each question that the user asked, is there a corresponding answer to it?\n", + " Question 1: Y\n", + "- Of the number of questions asked, how many of these questions were addressed by the answer? (output a number)\n", + "1\n" + ] + } + ], + "source": [ + "evaluation_output = eval_with_rubric(cust_prod_info, assistant_answer)\n", + "print(evaluation_output)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "'''中文Prompt'''\n", + "# 使用 GPT API 评估生成的回答\n", + "def eval_with_rubric(test_set, assistant_answer):\n", + "\n", + " cust_msg = test_set['customer_msg']\n", + " context = test_set['context']\n", + " completion = assistant_answer\n", + " \n", + " # 人设\n", + " system_message = \"\"\"\\\n", + " 你是一位助理，通过查看客户服务代理使用的上下文来评估客户服务代理回答用户问题的情况。\n", + " \"\"\"\n", + "\n", + " # 具体指令\n", + " user_message = f\"\"\"\\\n", + " 你正在根据代理使用的上下文评估对问题的提交答案。以下是数据：\n", + " [开始]\n", + " ************\n", + " [用户问题]: {cust_msg}\n", + " ************\n", + " [使用的上下文]: {context}\n", + " ************\n", + " [客户代理的回答]: {completion}\n", + " ************\n", + " [结束]\n", + "\n", + " 请将提交的答案的事实内容与上下文进行比较，忽略样式、语法或标点符号上的差异。\n", + " 回答以下问题：\n", + " 助手的回应是否只基于所提供的上下文？（是或否）\n", + " 回答中是否包含上下文中未提供的信息？（是或否）\n", + " 回应与上下文之间是否存在任何不一致之处？（是或否）\n", + " 计算用户提出了多少个问题。（输出一个数字）\n", + " 对于用户提出的每个问题，是否有相应的回答？\n", + " 问题1：（是或否）\n", + " 问题2：（是或否）\n", + " ...\n", + " 问题N：（是或否）\n", + " 在提出的问题数量中，有多少个问题在回答中得到了回应？（输出一个数字）\n", + "\"\"\"\n", + "\n", + " messages = [\n", + " {'role': 'system', 'content': system_message},\n", + " {'role': 'user', 'content': user_message}\n", + " ]\n", + "\n", + " response = get_completion_from_messages(messages)\n", + " return response" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "助手的回应是基于所提供的上下文。回答中没有包含上下文中未提供的信息。回应与上下文之间没有任何不一致之处。\n", + "\n", + "用户提出了两个问题。\n", + "\n", + "对于用户提出的每个问题，都有相应的回答。\n", + "\n", + "问题1：是\n", + "问题2：是\n", + "\n", + "在提出的问题数量中，所有问题都在回答中得到了回应，因此输出数字为2。\n" + ] + } + ], + "source": [ + "evaluation_output = eval_with_rubric(cust_prod_info, assistant_answer)\n", + "print(evaluation_output)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "给出一个标准回答，要求其评估生成回答与标准回答的差距" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "test_set_ideal = {\n", + " 'customer_msg': \"\"\"\\\n", + "tell me about the smartx pro phone and the fotosnap camera, the dslr one.\n", + "Also, what TVs or TV related products do you have?\"\"\",\n", + " 'ideal_answer':\"\"\"\\\n", + "Of course! The SmartX ProPhone is a powerful \\\n", + "smartphone with advanced camera features. \\\n", + "For instance, it has a 12MP dual camera. \\\n", + "Other features include 5G wireless and 128GB storage. \\\n", + "It also has a 6.1-inch display. The price is $899.99.\n", + "\n", + "The FotoSnap DSLR Camera is great for \\\n", + "capturing stunning photos and videos. \\\n", + "Some features include 1080p video, \\\n", + "3-inch LCD, a 24.2MP sensor, \\\n", + "and interchangeable lenses. \\\n", + "The price is 599.99.\n", + "\n", + "For TVs and TV related products, we offer 3 TVs \\\n", + "\n", + "\n", + "All TVs offer HDR and Smart TV.\n", + "\n", + "The CineView 4K TV has vibrant colors and smart features. \\\n", + "Some of these features include a 55-inch display, \\\n", + "'4K resolution. It's priced at 599.\n", + "\n", + "The CineView 8K TV is a stunning 8K TV. \\\n", + "Some features include a 65-inch display and \\\n", + "8K resolution. It's priced at 2999.99\n", + "\n", + "The CineView OLED TV lets you experience vibrant colors. \\\n", + "Some features include a 55-inch display and 4K resolution. \\\n", + "It's priced at 1499.99.\n", + "\n", + "We also offer 2 home theater products, both which include bluetooth.\\\n", + "The SoundMax Home Theater is a powerful home theater system for \\\n", + "an immmersive audio experience.\n", + "Its features include 5.1 channel, 1000W output, and wireless subwoofer.\n", + "It's priced at 399.99.\n", + "\n", + "The SoundMax Soundbar is a sleek and powerful soundbar.\n", + "It's features include 2.1 channel, 300W output, and wireless subwoofer.\n", + "It's priced at 199.99\n", + "\n", + "Are there any questions additional you may have about these products \\\n", + "that you mentioned here?\n", + "Or may do you have other questions I can help you with?\n", + " \"\"\"\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def eval_vs_ideal(test_set, assistant_answer):\n", + "\n", + " cust_msg = test_set['customer_msg']\n", + " ideal = test_set['ideal_answer']\n", + " completion = assistant_answer\n", + " \n", + " system_message = \"\"\"\\\n", + " You are an assistant that evaluates how well the customer service agent \\\n", + " answers a user question by comparing the response to the ideal (expert) response\n", + " Output a single letter and nothing else. \n", + " \"\"\"\n", + "\n", + " user_message = f\"\"\"\\\n", + "You are comparing a submitted answer to an expert answer on a given question. Here is the data:\n", + " [BEGIN DATA]\n", + " ************\n", + " [Question]: {cust_msg}\n", + " ************\n", + " [Expert]: {ideal}\n", + " ************\n", + " [Submission]: {completion}\n", + " ************\n", + " [END DATA]\n", + "\n", + "Compare the factual content of the submitted answer with the expert answer. Ignore any differences in style, grammar, or punctuation.\n", + " The submitted answer may either be a subset or superset of the expert answer, or it may conflict with it. Determine which case applies. Answer the question by selecting one of the following options:\n", + " (A) The submitted answer is a subset of the expert answer and is fully consistent with it.\n", + " (B) The submitted answer is a superset of the expert answer and is fully consistent with it.\n", + " (C) The submitted answer contains all the same details as the expert answer.\n", + " (D) There is a disagreement between the submitted answer and the expert answer.\n", + " (E) The answers differ, but these differences don't matter from the perspective of factuality.\n", + " choice_strings: ABCDE\n", + "\"\"\"\n", + "\n", + " messages = [\n", + " {'role': 'system', 'content': system_message},\n", + " {'role': 'user', 'content': user_message}\n", + " ]\n", + "\n", + " response = get_completion_from_messages(messages)\n", + " return response" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sure, I'd be happy to help! The SmartX ProPhone is a powerful smartphone with a 6.1-inch display, 128GB storage, 12MP dual camera, and 5G capabilities. The FotoSnap DSLR Camera is a versatile camera with a 24.2MP sensor, 1080p video, 3-inch LCD, and interchangeable lenses. As for TVs, we have a variety of options including the CineView 4K TV with a 55-inch display, 4K resolution, HDR, and smart TV capabilities, the CineView 8K TV with a 65-inch display, 8K resolution, HDR, and smart TV capabilities, and the CineView OLED TV with a 55-inch display, 4K resolution, HDR, and smart TV capabilities. We also have the SoundMax Home Theater system with 5.1 channel, 1000W output, wireless subwoofer, and Bluetooth, and the SoundMax Soundbar with 2.1 channel, 300W output, wireless subwoofer, and Bluetooth. Is there anything else I can help you with?\n" + ] + } + ], + "source": [ + "print(assistant_answer)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'A'" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "eval_vs_ideal(test_set_ideal, assistant_answer)\n", + "# 对于该生成回答，GPT 判断生成内容是标准回答的一个子集" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "assistant_answer_2 = \"life is like a box of chocolates\"" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'D'" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "eval_vs_ideal(test_set_ideal, assistant_answer_2)\n", + "# 对于明显异常答案，GPT 判断为不一致" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "'''基于中文Prompt的验证集'''\n", + "test_set_ideal = {\n", + " 'customer_msg': \"\"\"\\\n", + "告诉我有关 the smartx pro phone 和 the fotosnap camera, the dslr one 的信息。\\n另外，你们这有什么 TVs ？\"\"\",\n", + " 'ideal_answer':\"\"\"\\\n", + "SmartX ProPhone是一款功能强大的智能手机，具有6.1英寸的显示屏，128GB的存储空间，12MP双摄像头和5G网络。它的价格为899.99美元，保修期为1年。\n", + "FotoSnap DSLR相机是一款功能强大的相机，具有24.2MP传感器，1080p视频，3英寸LCD屏幕和可更换镜头。它的价格为599.99美元，保修期为1年。\n", + "我们有多种电视可供选择，包括CineView 4K电视，CineView 8K电视和CineView OLED电视。CineView 4K电视具有55英寸的显示屏，4K分辨率，HDR和智能电视功能，价格为599.99美元，保修期为2年。CineView 8K电视具有65英寸的显示屏，8K分辨率，HDR和智能电视功能，价格为2999.99美元，保修期为2年。CineView OLED电视具有55英寸的显示屏，4K分辨率，HDR和智能电视功能，价格为1499.99美元，保修期为2年\n", + " \"\"\"\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def eval_vs_ideal(test_set, assistant_answer):\n", + "\n", + " cust_msg = test_set['customer_msg']\n", + " ideal = test_set['ideal_answer']\n", + " completion = assistant_answer\n", + " \n", + " system_message = \"\"\"\\\n", + " 您是一位助理，通过将客户服务代理的回答与理想（专家）回答进行比较，评估客户服务代理对用户问题的回答质量。\n", + " 请输出一个单独的字母（A 、B、C、D、E），不要包含其他内容。 \n", + " \"\"\"\n", + "\n", + " user_message = f\"\"\"\\\n", + " 您正在比较一个给定问题的提交答案和专家答案。数据如下:\n", + " [开始]\n", + " ************\n", + " [问题]: {cust_msg}\n", + " ************\n", + " [专家答案]: {ideal}\n", + " ************\n", + " [提交答案]: {completion}\n", + " ************\n", + " [结束]\n", + "\n", + " 比较提交答案的事实内容与专家答案。忽略样式、语法或标点符号上的差异。\n", + " 提交的答案可能是专家答案的子集、超集，或者与之冲突。确定适用的情况，并通过选择以下选项之一回答问题：\n", + " （A）提交的答案是专家答案的子集，并且与之完全一致。\n", + " （B）提交的答案是专家答案的超集，并且与之完全一致。\n", + " （C）提交的答案包含与专家答案完全相同的细节。\n", + " （D）提交的答案与专家答案存在分歧。\n", + " （E）答案存在差异，但从事实的角度来看这些差异并不重要。\n", + " 选项：ABCDE\n", + "\"\"\"\n", + "\n", + " messages = [\n", + " {'role': 'system', 'content': system_message},\n", + " {'role': 'user', 'content': user_message}\n", + " ]\n", + "\n", + " response = get_completion_from_messages(messages)\n", + " return response" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "关于SmartX ProPhone和FotoSnap DSLR相机的信息：\n", + "\n", + "SmartX ProPhone是一款功能强大的智能手机，具有6.1英寸的显示屏，128GB的存储空间，12MP双摄像头和5G网络。它的价格为899.99美元，保修期为1年。\n", + "\n", + "FotoSnap DSLR相机是一款功能强大的相机，具有24.2MP传感器，1080p视频，3英寸LCD屏幕和可更换镜头。它的价格为599.99美元，保修期为1年。\n", + "\n", + "关于电视的信息：\n", + "\n", + "我们有多种电视可供选择，包括CineView 4K电视，CineView 8K电视和CineView OLED电视。CineView 4K电视具有55英寸的显示屏，4K分辨率，HDR和智能电视功能，价格为599.99美元，保修期为2年。CineView 8K电视具有65英寸的显示屏，8K分辨率，HDR和智能电视功能，价格为2999.99美元，保修期为2年。CineView OLED电视具有55英寸的显示屏，4K分辨率，HDR和智能电视功能，价格为1499.99美元，保修期为2年。您需要哪种类型的电视？\n" + ] + } + ], + "source": [ + "print(assistant_answer)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'B'" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "eval_vs_ideal(test_set_ideal, assistant_answer)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "assistant_answer_2 = \"life is like a box of chocolates\"" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'D'" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "eval_vs_ideal(test_set_ideal, assistant_answer_2)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "zyh_gpt", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +}