合并master
This commit is contained in:
w_xiaolizu
@ -1,35 +1,53 @@
|
||||
# 如何使用其他大语言模型（v3.0分支测试中）
|
||||
|
||||
## 1. 先运行text-generation
|
||||
## ChatGLM
|
||||
|
||||
- 安装依赖 `pip install -r request_llm/requirements_chatglm.txt`
|
||||
- 修改配置，在config.py中将LLM_MODEL的值改为"chatglm"
|
||||
|
||||
``` sh
|
||||
# 下载模型（ text-generation 这么牛的项目，别忘了给人家star ）
|
||||
LLM_MODEL = "chatglm"
|
||||
```
|
||||
- 运行！
|
||||
``` sh
|
||||
`python main.py`
|
||||
```
|
||||
|
||||
|
||||
---
|
||||
## Text-Generation-UI (TGUI)
|
||||
|
||||
### 1. 部署TGUI
|
||||
``` sh
|
||||
# 1 下载模型
|
||||
git clone https://github.com/oobabooga/text-generation-webui.git
|
||||
|
||||
# 安装text-generation的额外依赖
|
||||
pip install accelerate bitsandbytes flexgen gradio llamacpp markdown numpy peft requests rwkv safetensors sentencepiece tqdm datasets git+https://github.com/huggingface/transformers
|
||||
|
||||
# 切换路径
|
||||
# 2 这个仓库的最新代码有问题，回滚到几周之前
|
||||
git reset --hard fcda3f87767e642d1c0411776e549e1d3894843d
|
||||
# 3 切换路径
|
||||
cd text-generation-webui
|
||||
|
||||
# 下载模型
|
||||
# 4 安装text-generation的额外依赖
|
||||
pip install accelerate bitsandbytes flexgen gradio llamacpp markdown numpy peft requests rwkv safetensors sentencepiece tqdm datasets git+https://github.com/huggingface/transformers
|
||||
# 5 下载模型
|
||||
python download-model.py facebook/galactica-1.3b
|
||||
# 其他可选如 facebook/opt-1.3b
|
||||
# facebook/galactica-1.3b
|
||||
# facebook/galactica-6.7b
|
||||
# facebook/galactica-120b
|
||||
# facebook/pygmalion-1.3b 等
|
||||
# 详情见 https://github.com/oobabooga/text-generation-webui
|
||||
|
||||
# 启动text-generation，注意把模型的斜杠改成下划线
|
||||
python server.py --cpu --listen --listen-port 7860 --model facebook_galactica-1.3b
|
||||
# 6 启动text-generation
|
||||
python server.py --cpu --listen --listen-port 7865 --model facebook_galactica-1.3b
|
||||
```
|
||||
|
||||
## 2. 修改config.py
|
||||
### 2. 修改config.py
|
||||
|
||||
``` sh
|
||||
# LLM_MODEL格式较复杂 TGUI:[模型]@[ws地址]:[ws端口] , 端口要和上面给定的端口一致
|
||||
LLM_MODEL = "TGUI:galactica-1.3b@localhost:7860"
|
||||
# LLM_MODEL格式: tgui:[模型]@[ws地址]:[ws端口] , 端口要和上面给定的端口一致
|
||||
LLM_MODEL = "tgui:galactica-1.3b@localhost:7860"
|
||||
```
|
||||
|
||||
## 3. 运行！
|
||||
### 3. 运行！
|
||||
``` sh
|
||||
cd chatgpt-academic
|
||||
python main.py
|
||||
|
||||
210
request_llm/bridge_all.py
Normal file
210
request_llm/bridge_all.py
Normal file
@ -0,0 +1,210 @@
|
||||
|
||||
"""
|
||||
该文件中主要包含2个函数
|
||||
|
||||
不具备多线程能力的函数：
|
||||
1. predict: 正常对话时使用，具备完备的交互功能，不可多线程
|
||||
|
||||
具备多线程调用能力的函数
|
||||
2. predict_no_ui_long_connection：在实验过程中发现调用predict_no_ui处理长文档时，和openai的连接容易断掉，这个函数用stream的方式解决这个问题，同样支持多线程
|
||||
"""
|
||||
import tiktoken
|
||||
from functools import wraps, lru_cache
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from .bridge_chatgpt import predict_no_ui_long_connection as chatgpt_noui
|
||||
from .bridge_chatgpt import predict as chatgpt_ui
|
||||
|
||||
from .bridge_chatglm import predict_no_ui_long_connection as chatglm_noui
|
||||
from .bridge_chatglm import predict as chatglm_ui
|
||||
|
||||
# from .bridge_tgui import predict_no_ui_long_connection as tgui_noui
|
||||
# from .bridge_tgui import predict as tgui_ui
|
||||
|
||||
colors = ['#FF00FF', '#00FFFF', '#FF0000', '#990099', '#009999', '#990044']
|
||||
|
||||
class LazyloadTiktoken(object):
|
||||
def __init__(self, model):
|
||||
self.model = model
|
||||
|
||||
@staticmethod
|
||||
@lru_cache(maxsize=128)
|
||||
def get_encoder(model):
|
||||
print('正在加载tokenizer，如果是第一次运行，可能需要一点时间下载参数')
|
||||
tmp = tiktoken.encoding_for_model(model)
|
||||
print('加载tokenizer完毕')
|
||||
return tmp
|
||||
|
||||
def encode(self, *args, **kwargs):
|
||||
encoder = self.get_encoder(self.model)
|
||||
return encoder.encode(*args, **kwargs)
|
||||
|
||||
def decode(self, *args, **kwargs):
|
||||
encoder = self.get_encoder(self.model)
|
||||
return encoder.decode(*args, **kwargs)
|
||||
|
||||
tokenizer_gpt35 = LazyloadTiktoken("gpt-3.5-turbo")
|
||||
tokenizer_gpt4 = LazyloadTiktoken("gpt-4")
|
||||
get_token_num_gpt35 = lambda txt: len(tokenizer_gpt35.encode(txt, disallowed_special=()))
|
||||
get_token_num_gpt4 = lambda txt: len(tokenizer_gpt4.encode(txt, disallowed_special=()))
|
||||
|
||||
model_info = {
|
||||
# openai
|
||||
"gpt-3.5-turbo": {
|
||||
"fn_with_ui": chatgpt_ui,
|
||||
"fn_without_ui": chatgpt_noui,
|
||||
"endpoint": "https://api.openai.com/v1/chat/completions",
|
||||
"max_token": 4096,
|
||||
"tokenizer": tokenizer_gpt35,
|
||||
"token_cnt": get_token_num_gpt35,
|
||||
},
|
||||
|
||||
"gpt-4": {
|
||||
"fn_with_ui": chatgpt_ui,
|
||||
"fn_without_ui": chatgpt_noui,
|
||||
"endpoint": "https://api.openai.com/v1/chat/completions",
|
||||
"max_token": 8192,
|
||||
"tokenizer": tokenizer_gpt4,
|
||||
"token_cnt": get_token_num_gpt4,
|
||||
},
|
||||
|
||||
# api_2d
|
||||
"api2d-gpt-3.5-turbo": {
|
||||
"fn_with_ui": chatgpt_ui,
|
||||
"fn_without_ui": chatgpt_noui,
|
||||
"endpoint": "https://openai.api2d.net/v1/chat/completions",
|
||||
"max_token": 4096,
|
||||
"tokenizer": tokenizer_gpt35,
|
||||
"token_cnt": get_token_num_gpt35,
|
||||
},
|
||||
|
||||
"api2d-gpt-4": {
|
||||
"fn_with_ui": chatgpt_ui,
|
||||
"fn_without_ui": chatgpt_noui,
|
||||
"endpoint": "https://openai.api2d.net/v1/chat/completions",
|
||||
"max_token": 8192,
|
||||
"tokenizer": tokenizer_gpt4,
|
||||
"token_cnt": get_token_num_gpt4,
|
||||
},
|
||||
|
||||
# chatglm
|
||||
"chatglm": {
|
||||
"fn_with_ui": chatglm_ui,
|
||||
"fn_without_ui": chatglm_noui,
|
||||
"endpoint": None,
|
||||
"max_token": 1024,
|
||||
"tokenizer": tokenizer_gpt35,
|
||||
"token_cnt": get_token_num_gpt35,
|
||||
},
|
||||
|
||||
}
|
||||
|
||||
|
||||
def LLM_CATCH_EXCEPTION(f):
|
||||
"""
|
||||
装饰器函数，将错误显示出来
|
||||
"""
|
||||
def decorated(inputs, llm_kwargs, history, sys_prompt, observe_window, console_slience):
|
||||
try:
|
||||
return f(inputs, llm_kwargs, history, sys_prompt, observe_window, console_slience)
|
||||
except Exception as e:
|
||||
from toolbox import get_conf
|
||||
import traceback
|
||||
proxies, = get_conf('proxies')
|
||||
tb_str = '\n```\n' + traceback.format_exc() + '\n```\n'
|
||||
observe_window[0] = tb_str
|
||||
return tb_str
|
||||
return decorated
|
||||
|
||||
|
||||
def predict_no_ui_long_connection(inputs, llm_kwargs, history, sys_prompt, observe_window, console_slience=False):
|
||||
"""
|
||||
发送至LLM，等待回复，一次性完成，不显示中间过程。但内部用stream的方法避免中途网线被掐。
|
||||
inputs：
|
||||
是本次问询的输入
|
||||
sys_prompt:
|
||||
系统静默prompt
|
||||
llm_kwargs：
|
||||
LLM的内部调优参数
|
||||
history：
|
||||
是之前的对话列表
|
||||
observe_window = None：
|
||||
用于负责跨越线程传递已经输出的部分，大部分时候仅仅为了fancy的视觉效果，留空即可。observe_window[0]：观测窗。observe_window[1]：看门狗
|
||||
"""
|
||||
import threading, time, copy
|
||||
|
||||
model = llm_kwargs['llm_model']
|
||||
n_model = 1
|
||||
if '&' not in model:
|
||||
assert not model.startswith("tgui"), "TGUI不支持函数插件的实现"
|
||||
|
||||
# 如果只询问1个大语言模型：
|
||||
method = model_info[model]["fn_without_ui"]
|
||||
return method(inputs, llm_kwargs, history, sys_prompt, observe_window, console_slience)
|
||||
else:
|
||||
# 如果同时询问多个大语言模型：
|
||||
executor = ThreadPoolExecutor(max_workers=4)
|
||||
models = model.split('&')
|
||||
n_model = len(models)
|
||||
|
||||
window_len = len(observe_window)
|
||||
assert window_len==3
|
||||
window_mutex = [["", time.time(), ""] for _ in range(n_model)] + [True]
|
||||
|
||||
futures = []
|
||||
for i in range(n_model):
|
||||
model = models[i]
|
||||
method = model_info[model]["fn_without_ui"]
|
||||
llm_kwargs_feedin = copy.deepcopy(llm_kwargs)
|
||||
llm_kwargs_feedin['llm_model'] = model
|
||||
future = executor.submit(LLM_CATCH_EXCEPTION(method), inputs, llm_kwargs_feedin, history, sys_prompt, window_mutex[i], console_slience)
|
||||
futures.append(future)
|
||||
|
||||
def mutex_manager(window_mutex, observe_window):
|
||||
while True:
|
||||
time.sleep(0.5)
|
||||
if not window_mutex[-1]: break
|
||||
# 看门狗（watchdog）
|
||||
for i in range(n_model):
|
||||
window_mutex[i][1] = observe_window[1]
|
||||
# 观察窗（window）
|
||||
chat_string = []
|
||||
for i in range(n_model):
|
||||
chat_string.append( f"【{str(models[i])} 说】: <font color=\"{colors[i]}\"> {window_mutex[i][0]} </font>" )
|
||||
res = '<br/><br/>\n\n---\n\n'.join(chat_string)
|
||||
# # # # # # # # # # #
|
||||
observe_window[0] = res
|
||||
|
||||
t_model = threading.Thread(target=mutex_manager, args=(window_mutex, observe_window), daemon=True)
|
||||
t_model.start()
|
||||
|
||||
return_string_collect = []
|
||||
while True:
|
||||
worker_done = [h.done() for h in futures]
|
||||
if all(worker_done):
|
||||
executor.shutdown()
|
||||
break
|
||||
time.sleep(1)
|
||||
|
||||
for i, future in enumerate(futures): # wait and get
|
||||
return_string_collect.append( f"【{str(models[i])} 说】: <font color=\"{colors[i]}\"> {future.result()} </font>" )
|
||||
|
||||
window_mutex[-1] = False # stop mutex thread
|
||||
res = '<br/>\n\n---\n\n'.join(return_string_collect)
|
||||
return res
|
||||
|
||||
|
||||
def predict(inputs, llm_kwargs, *args, **kwargs):
|
||||
"""
|
||||
发送至LLM，流式获取输出。
|
||||
用于基础的对话功能。
|
||||
inputs 是本次问询的输入
|
||||
top_p, temperature是LLM的内部调优参数
|
||||
history 是之前的对话列表（注意无论是inputs还是history，内容太长了都会触发token数量溢出的错误）
|
||||
chatbot 为WebUI中显示的对话列表，修改它，然后yeild出去，可以直接修改对话界面内容
|
||||
additional_fn代表点击的哪个按钮，按钮见functional.py
|
||||
"""
|
||||
|
||||
method = model_info[llm_kwargs['llm_model']]["fn_with_ui"]
|
||||
yield from method(inputs, llm_kwargs, *args, **kwargs)
|
||||
|
||||
140
request_llm/bridge_chatglm.py
Normal file
140
request_llm/bridge_chatglm.py
Normal file
@ -0,0 +1,140 @@
|
||||
|
||||
from transformers import AutoModel, AutoTokenizer
|
||||
import time
|
||||
import importlib
|
||||
from toolbox import update_ui, get_conf
|
||||
from multiprocessing import Process, Pipe
|
||||
|
||||
load_message = "ChatGLM尚未加载，加载需要一段时间。注意，取决于`config.py`的配置，ChatGLM消耗大量的内存（CPU）或显存（GPU），也许会导致低配计算机卡死 ……"
|
||||
|
||||
#################################################################################
|
||||
class GetGLMHandle(Process):
|
||||
def __init__(self):
|
||||
super().__init__(daemon=True)
|
||||
self.parent, self.child = Pipe()
|
||||
self.chatglm_model = None
|
||||
self.chatglm_tokenizer = None
|
||||
self.info = ""
|
||||
self.success = True
|
||||
self.check_dependency()
|
||||
self.start()
|
||||
|
||||
def check_dependency(self):
|
||||
try:
|
||||
import sentencepiece
|
||||
self.info = "依赖检测通过"
|
||||
self.success = True
|
||||
except:
|
||||
self.info = "缺少ChatGLM的依赖，如果要使用ChatGLM，除了基础的pip依赖以外，您还需要运行`pip install -r request_llm/requirements_chatglm.txt`安装ChatGLM的依赖。"
|
||||
self.success = False
|
||||
|
||||
def ready(self):
|
||||
return self.chatglm_model is not None
|
||||
|
||||
def run(self):
|
||||
# 第一次运行，加载参数
|
||||
retry = 0
|
||||
while True:
|
||||
try:
|
||||
if self.chatglm_model is None:
|
||||
self.chatglm_tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True)
|
||||
device, = get_conf('LOCAL_MODEL_DEVICE')
|
||||
if device=='cpu':
|
||||
self.chatglm_model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).float()
|
||||
else:
|
||||
self.chatglm_model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda()
|
||||
self.chatglm_model = self.chatglm_model.eval()
|
||||
break
|
||||
else:
|
||||
break
|
||||
except:
|
||||
retry += 1
|
||||
if retry > 3:
|
||||
self.child.send('[Local Message] Call ChatGLM fail 不能正常加载ChatGLM的参数。')
|
||||
raise RuntimeError("不能正常加载ChatGLM的参数！")
|
||||
|
||||
# 进入任务等待状态
|
||||
while True:
|
||||
kwargs = self.child.recv()
|
||||
try:
|
||||
for response, history in self.chatglm_model.stream_chat(self.chatglm_tokenizer, **kwargs):
|
||||
self.child.send(response)
|
||||
except:
|
||||
self.child.send('[Local Message] Call ChatGLM fail.')
|
||||
self.child.send('[Finish]')
|
||||
|
||||
def stream_chat(self, **kwargs):
|
||||
self.parent.send(kwargs)
|
||||
while True:
|
||||
res = self.parent.recv()
|
||||
if res != '[Finish]':
|
||||
yield res
|
||||
else:
|
||||
break
|
||||
return
|
||||
|
||||
global glm_handle
|
||||
glm_handle = None
|
||||
#################################################################################
|
||||
def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=None, console_slience=False):
|
||||
"""
|
||||
多线程方法
|
||||
函数的说明请见 request_llm/bridge_all.py
|
||||
"""
|
||||
global glm_handle
|
||||
if glm_handle is None:
|
||||
glm_handle = GetGLMHandle()
|
||||
observe_window[0] = load_message + "\n\n" + glm_handle.info
|
||||
if not glm_handle.success:
|
||||
error = glm_handle.info
|
||||
glm_handle = None
|
||||
raise RuntimeError(error)
|
||||
|
||||
# chatglm 没有 sys_prompt 接口，因此把prompt加入 history
|
||||
history_feedin = []
|
||||
for i in range(len(history)//2):
|
||||
history_feedin.append(["What can I do?", sys_prompt] )
|
||||
history_feedin.append([history[2*i], history[2*i+1]] )
|
||||
|
||||
watch_dog_patience = 5 # 看门狗 (watchdog) 的耐心, 设置5秒即可
|
||||
response = ""
|
||||
for response in glm_handle.stream_chat(query=inputs, history=history_feedin, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
|
||||
observe_window[0] = response
|
||||
if len(observe_window) >= 2:
|
||||
if (time.time()-observe_window[1]) > watch_dog_patience:
|
||||
raise RuntimeError("程序终止。")
|
||||
return response
|
||||
|
||||
|
||||
|
||||
def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
|
||||
"""
|
||||
单线程方法
|
||||
函数的说明请见 request_llm/bridge_all.py
|
||||
"""
|
||||
chatbot.append((inputs, ""))
|
||||
|
||||
global glm_handle
|
||||
if glm_handle is None:
|
||||
glm_handle = GetGLMHandle()
|
||||
chatbot[-1] = (inputs, load_message + "\n\n" + glm_handle.info)
|
||||
yield from update_ui(chatbot=chatbot, history=[])
|
||||
if not glm_handle.success:
|
||||
glm_handle = None
|
||||
return
|
||||
|
||||
if additional_fn is not None:
|
||||
import core_functional
|
||||
importlib.reload(core_functional) # 热更新prompt
|
||||
core_functional = core_functional.get_core_functions()
|
||||
if "PreProcess" in core_functional[additional_fn]: inputs = core_functional[additional_fn]["PreProcess"](inputs) # 获取预处理函数（如果有的话）
|
||||
inputs = core_functional[additional_fn]["Prefix"] + inputs + core_functional[additional_fn]["Suffix"]
|
||||
|
||||
history_feedin = []
|
||||
for i in range(len(history)//2):
|
||||
history_feedin.append(["What can I do?", system_prompt] )
|
||||
history_feedin.append([history[2*i], history[2*i+1]] )
|
||||
|
||||
for response in glm_handle.stream_chat(query=inputs, history=history_feedin, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
|
||||
chatbot[-1] = (inputs, response)
|
||||
yield from update_ui(chatbot=chatbot, history=history)
|
||||
@ -21,9 +21,9 @@ import importlib
|
||||
|
||||
# config_private.py放自己的秘密如API和代理网址
|
||||
# 读取时首先看是否存在私密的config_private配置文件（不受git管控），如果有，则覆盖原config文件
|
||||
from toolbox import get_conf, update_ui
|
||||
proxies, API_URL, API_KEY, TIMEOUT_SECONDS, MAX_RETRY, LLM_MODEL = \
|
||||
get_conf('proxies', 'API_URL', 'API_KEY', 'TIMEOUT_SECONDS', 'MAX_RETRY', 'LLM_MODEL')
|
||||
from toolbox import get_conf, update_ui, is_any_api_key, select_api_key
|
||||
proxies, API_KEY, TIMEOUT_SECONDS, MAX_RETRY = \
|
||||
get_conf('proxies', 'API_KEY', 'TIMEOUT_SECONDS', 'MAX_RETRY')
|
||||
|
||||
timeout_bot_msg = '[Local Message] Request timeout. Network error. Please check proxy settings in config.py.' + \
|
||||
'网络错误，检查代理服务器是否可用，以及代理设置的格式是否正确，格式须是[协议]://[地址]:[端口]，缺一不可。'
|
||||
@ -42,25 +42,27 @@ def get_full_error(chunk, stream_response):
|
||||
|
||||
def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=None, console_slience=False):
|
||||
"""
|
||||
发送至chatGPT，等待回复，一次性完成，不显示中间过程。但内部用stream的方法避免中途网线被掐。
|
||||
inputs：
|
||||
是本次问询的输入
|
||||
sys_prompt:
|
||||
系统静默prompt
|
||||
llm_kwargs：
|
||||
chatGPT的内部调优参数
|
||||
history：
|
||||
是之前的对话列表
|
||||
observe_window = None：
|
||||
用于负责跨越线程传递已经输出的部分，大部分时候仅仅为了fancy的视觉效果，留空即可。observe_window[0]：观测窗。observe_window[1]：看门狗
|
||||
发送至chatGPT，等待回复，一次性完成，不显示中间过程。但内部用stream的方法避免中途网线被掐。
|
||||
inputs：
|
||||
是本次问询的输入
|
||||
sys_prompt:
|
||||
系统静默prompt
|
||||
llm_kwargs：
|
||||
chatGPT的内部调优参数
|
||||
history：
|
||||
是之前的对话列表
|
||||
observe_window = None：
|
||||
用于负责跨越线程传递已经输出的部分，大部分时候仅仅为了fancy的视觉效果，留空即可。observe_window[0]：观测窗。observe_window[1]：看门狗
|
||||
"""
|
||||
watch_dog_patience = 5 # 看门狗的耐心, 设置5秒即可
|
||||
headers, payload = generate_payload(inputs, llm_kwargs, history, system_prompt=sys_prompt, stream=True)
|
||||
retry = 0
|
||||
while True:
|
||||
try:
|
||||
# make a POST requests to the API endpoint, stream=False
|
||||
response = requests.post(API_URL, headers=headers, proxies=proxies,
|
||||
# make a POST request to the API endpoint, stream=False
|
||||
from .bridge_all import model_info
|
||||
endpoint = model_info[llm_kwargs['llm_model']]['endpoint']
|
||||
response = requests.post(endpoint, headers=headers, proxies=proxies,
|
||||
json=payload, stream=True, timeout=TIMEOUT_SECONDS); break
|
||||
except requests.exceptions.ReadTimeout as e:
|
||||
retry += 1
|
||||
@ -83,6 +85,7 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="",
|
||||
raise ConnectionAbortedError("OpenAI拒绝了请求:" + error_msg)
|
||||
else:
|
||||
raise RuntimeError("OpenAI拒绝了请求：" + error_msg)
|
||||
if ('data: [DONE]' in chunk): break # api2d 正常完成
|
||||
json_data = json.loads(chunk.lstrip('data:'))['choices'][0]
|
||||
delta = json_data["delta"]
|
||||
if len(delta) == 0: break
|
||||
@ -105,22 +108,22 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="",
|
||||
|
||||
def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
|
||||
"""
|
||||
发送至chatGPT，流式获取输出。
|
||||
用于基础的对话功能。
|
||||
inputs 是本次问询的输入
|
||||
top_p, temperature是chatGPT的内部调优参数
|
||||
history 是之前的对话列表（注意无论是inputs还是history，内容太长了都会触发token数量溢出的错误）
|
||||
chatbot 为WebUI中显示的对话列表，修改它，然后yeild出去，可以直接修改对话界面内容
|
||||
additional_fn代表点击的哪个按钮，按钮见functional.py
|
||||
发送至chatGPT，流式获取输出。
|
||||
用于基础的对话功能。
|
||||
inputs 是本次问询的输入
|
||||
top_p, temperature是chatGPT的内部调优参数
|
||||
history 是之前的对话列表（注意无论是inputs还是history，内容太长了都会触发token数量溢出的错误）
|
||||
chatbot 为WebUI中显示的对话列表，修改它，然后yeild出去，可以直接修改对话界面内容
|
||||
additional_fn代表点击的哪个按钮，按钮见functional.py
|
||||
"""
|
||||
if inputs.startswith('sk-') and len(inputs) == 51:
|
||||
if is_any_api_key(inputs):
|
||||
chatbot._cookies['api_key'] = inputs
|
||||
chatbot.append(("输入已识别为openai的api_key", "api_key已导入"))
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg="api_key已导入") # 刷新界面
|
||||
return
|
||||
elif len(chatbot._cookies['api_key']) != 51:
|
||||
elif not is_any_api_key(chatbot._cookies['api_key']):
|
||||
chatbot.append((inputs, "缺少api_key。\n\n1. 临时解决方案：直接在输入区键入api_key，然后回车提交。\n\n2. 长效解决方案：在config.py中配置。"))
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg="api_key已导入") # 刷新界面
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg="缺少api_key") # 刷新界面
|
||||
return
|
||||
|
||||
if additional_fn is not None:
|
||||
@ -130,20 +133,27 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp
|
||||
if "PreProcess" in core_functional[additional_fn]: inputs = core_functional[additional_fn]["PreProcess"](inputs) # 获取预处理函数（如果有的话）
|
||||
inputs = core_functional[additional_fn]["Prefix"] + inputs + core_functional[additional_fn]["Suffix"]
|
||||
|
||||
if stream:
|
||||
raw_input = inputs
|
||||
logging.info(f'[raw_input] {raw_input}')
|
||||
chatbot.append((inputs, ""))
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg="等待响应") # 刷新界面
|
||||
raw_input = inputs
|
||||
logging.info(f'[raw_input] {raw_input}')
|
||||
chatbot.append((inputs, ""))
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg="等待响应") # 刷新界面
|
||||
|
||||
headers, payload = generate_payload(inputs, llm_kwargs, history, system_prompt, stream)
|
||||
try:
|
||||
headers, payload = generate_payload(inputs, llm_kwargs, history, system_prompt, stream)
|
||||
except RuntimeError as e:
|
||||
chatbot[-1] = (inputs, f"您提供的api-key不满足要求，不包含任何可用于{llm_kwargs['llm_model']}的api-key。")
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg="api-key不满足要求") # 刷新界面
|
||||
return
|
||||
|
||||
history.append(inputs); history.append(" ")
|
||||
|
||||
retry = 0
|
||||
while True:
|
||||
try:
|
||||
# make a POST requests to the API endpoint, stream=True
|
||||
response = requests.post(API_URL, headers=headers, proxies=proxies,
|
||||
# make a POST request to the API endpoint, stream=True
|
||||
from .bridge_all import model_info
|
||||
endpoint = model_info[llm_kwargs['llm_model']]['endpoint']
|
||||
response = requests.post(endpoint, headers=headers, proxies=proxies,
|
||||
json=payload, stream=True, timeout=TIMEOUT_SECONDS);break
|
||||
except:
|
||||
retry += 1
|
||||
@ -160,21 +170,23 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp
|
||||
while True:
|
||||
chunk = next(stream_response)
|
||||
# print(chunk.decode()[6:])
|
||||
if is_head_of_the_stream:
|
||||
if is_head_of_the_stream and (r'"object":"error"' not in chunk.decode()):
|
||||
# 数据流的第一帧不携带content
|
||||
is_head_of_the_stream = False; continue
|
||||
|
||||
if chunk:
|
||||
try:
|
||||
if len(json.loads(chunk.decode()[6:])['choices'][0]["delta"]) == 0:
|
||||
chunk_decoded = chunk.decode()
|
||||
# 前者API2D的
|
||||
if ('data: [DONE]' in chunk_decoded) or (len(json.loads(chunk_decoded[6:])['choices'][0]["delta"]) == 0):
|
||||
# 判定为数据流的结束，gpt_replying_buffer也写完了
|
||||
logging.info(f'[response] {gpt_replying_buffer}')
|
||||
break
|
||||
# 处理数据流的主体
|
||||
chunkjson = json.loads(chunk.decode()[6:])
|
||||
chunkjson = json.loads(chunk_decoded[6:])
|
||||
status_text = f"finish_reason: {chunkjson['choices'][0]['finish_reason']}"
|
||||
# 如果这里抛出异常，一般是文本过长，详情见get_full_error的输出
|
||||
gpt_replying_buffer = gpt_replying_buffer + json.loads(chunk.decode()[6:])['choices'][0]["delta"]["content"]
|
||||
gpt_replying_buffer = gpt_replying_buffer + json.loads(chunk_decoded[6:])['choices'][0]["delta"]["content"]
|
||||
history[-1] = gpt_replying_buffer
|
||||
chatbot[-1] = (history[-2], history[-1])
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg=status_text) # 刷新界面
|
||||
@ -183,31 +195,38 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp
|
||||
traceback.print_exc()
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg="Json解析不合常规") # 刷新界面
|
||||
chunk = get_full_error(chunk, stream_response)
|
||||
error_msg = chunk.decode()
|
||||
chunk_decoded = chunk.decode()
|
||||
error_msg = chunk_decoded
|
||||
if "reduce the length" in error_msg:
|
||||
chatbot[-1] = (chatbot[-1][0], "[Local Message] Reduce the length. 本次输入过长，或历史数据过长. 历史缓存数据现已释放，您可以请再次尝试.")
|
||||
history = [] # 清除历史
|
||||
elif "does not exist" in error_msg:
|
||||
chatbot[-1] = (chatbot[-1][0], f"[Local Message] Model {llm_kwargs['llm_model']} does not exist. 模型不存在，或者您没有获得体验资格.")
|
||||
elif "Incorrect API key" in error_msg:
|
||||
chatbot[-1] = (chatbot[-1][0], "[Local Message] Incorrect API key. OpenAI以提供了不正确的API_KEY为由，拒绝服务.")
|
||||
elif "exceeded your current quota" in error_msg:
|
||||
chatbot[-1] = (chatbot[-1][0], "[Local Message] You exceeded your current quota. OpenAI以账户额度不足为由，拒绝服务.")
|
||||
elif "bad forward key" in error_msg:
|
||||
chatbot[-1] = (chatbot[-1][0], "[Local Message] Bad forward key. API2D账户额度不足.")
|
||||
else:
|
||||
from toolbox import regular_txt_to_markdown
|
||||
tb_str = '```\n' + traceback.format_exc() + '```'
|
||||
chatbot[-1] = (chatbot[-1][0], f"[Local Message] 异常 \n\n{tb_str} \n\n{regular_txt_to_markdown(chunk.decode()[4:])}")
|
||||
chatbot[-1] = (chatbot[-1][0], f"[Local Message] 异常 \n\n{tb_str} \n\n{regular_txt_to_markdown(chunk_decoded[4:])}")
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg="Json异常" + error_msg) # 刷新界面
|
||||
return
|
||||
|
||||
def generate_payload(inputs, llm_kwargs, history, system_prompt, stream):
|
||||
"""
|
||||
整合所有信息，选择LLM模型，生成http请求，为发送请求做准备
|
||||
整合所有信息，选择LLM模型，生成http请求，为发送请求做准备
|
||||
"""
|
||||
if len(llm_kwargs['api_key']) != 51:
|
||||
if not is_any_api_key(llm_kwargs['api_key']):
|
||||
raise AssertionError("你提供了错误的API_KEY。\n\n1. 临时解决方案：直接在输入区键入api_key，然后回车提交。\n\n2. 长效解决方案：在config.py中配置。")
|
||||
|
||||
api_key = select_api_key(llm_kwargs['api_key'], llm_kwargs['llm_model'])
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {llm_kwargs['api_key']}"
|
||||
"Authorization": f"Bearer {api_key}"
|
||||
}
|
||||
|
||||
conversation_cnt = len(history) // 2
|
||||
@ -235,7 +254,7 @@ def generate_payload(inputs, llm_kwargs, history, system_prompt, stream):
|
||||
messages.append(what_i_ask_now)
|
||||
|
||||
payload = {
|
||||
"model": llm_kwargs['llm_model'],
|
||||
"model": llm_kwargs['llm_model'].strip('api2d-'),
|
||||
"messages": messages,
|
||||
"temperature": llm_kwargs['temperature'], # 1.0,
|
||||
"top_p": llm_kwargs['top_p'], # 1.0,
|
||||
|
||||
@ -13,23 +13,18 @@ import time
|
||||
import threading
|
||||
import importlib
|
||||
from toolbox import get_conf, update_ui
|
||||
LLM_MODEL, = get_conf('LLM_MODEL')
|
||||
|
||||
# "TGUI:galactica-1.3b@localhost:7860"
|
||||
model_name, addr_port = LLM_MODEL.split('@')
|
||||
assert ':' in addr_port, "LLM_MODEL 格式不正确！" + LLM_MODEL
|
||||
addr, port = addr_port.split(':')
|
||||
|
||||
def random_hash():
|
||||
letters = string.ascii_lowercase + string.digits
|
||||
return ''.join(random.choice(letters) for i in range(9))
|
||||
|
||||
async def run(context, max_token=512):
|
||||
async def run(context, max_token, temperature, top_p, addr, port):
|
||||
params = {
|
||||
'max_new_tokens': max_token,
|
||||
'do_sample': True,
|
||||
'temperature': 0.5,
|
||||
'top_p': 0.9,
|
||||
'temperature': temperature,
|
||||
'top_p': top_p,
|
||||
'typical_p': 1,
|
||||
'repetition_penalty': 1.05,
|
||||
'encoder_repetition_penalty': 1.0,
|
||||
@ -90,7 +85,7 @@ async def run(context, max_token=512):
|
||||
|
||||
|
||||
|
||||
def predict_tgui(inputs, top_p, temperature, chatbot, history=[], system_prompt='' , stream = True, additional_fn=None):
|
||||
def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
|
||||
"""
|
||||
发送至chatGPT，流式获取输出。
|
||||
用于基础的对话功能。
|
||||
@ -108,18 +103,26 @@ def predict_tgui(inputs, top_p, temperature, chatbot, history=[], system_prompt=
|
||||
inputs = core_functional[additional_fn]["Prefix"] + inputs + core_functional[additional_fn]["Suffix"]
|
||||
|
||||
raw_input = "What I would like to say is the following: " + inputs
|
||||
logging.info(f'[raw_input] {raw_input}')
|
||||
history.extend([inputs, ""])
|
||||
chatbot.append([inputs, ""])
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg="等待响应") # 刷新界面
|
||||
|
||||
prompt = inputs
|
||||
prompt = raw_input
|
||||
tgui_say = ""
|
||||
|
||||
model_name, addr_port = llm_kwargs['llm_model'].split('@')
|
||||
assert ':' in addr_port, "LLM_MODEL 格式不正确！" + llm_kwargs['llm_model']
|
||||
addr, port = addr_port.split(':')
|
||||
|
||||
|
||||
mutable = ["", time.time()]
|
||||
def run_coorotine(mutable):
|
||||
async def get_result(mutable):
|
||||
async for response in run(prompt):
|
||||
# "tgui:galactica-1.3b@localhost:7860"
|
||||
|
||||
async for response in run(context=prompt, max_token=llm_kwargs['max_length'],
|
||||
temperature=llm_kwargs['temperature'],
|
||||
top_p=llm_kwargs['top_p'], addr=addr, port=port):
|
||||
print(response[len(mutable[0]):])
|
||||
mutable[0] = response
|
||||
if (time.time() - mutable[1]) > 3:
|
||||
@ -140,28 +143,29 @@ def predict_tgui(inputs, top_p, temperature, chatbot, history=[], system_prompt=
|
||||
chatbot[-1] = (history[-2], history[-1])
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
logging.info(f'[response] {tgui_say}')
|
||||
|
||||
|
||||
|
||||
def predict_tgui_no_ui(inputs, top_p, temperature, history=[], sys_prompt=""):
|
||||
def predict_no_ui_long_connection(inputs, llm_kwargs, history, sys_prompt, observe_window, console_slience=False):
|
||||
raw_input = "What I would like to say is the following: " + inputs
|
||||
prompt = inputs
|
||||
prompt = raw_input
|
||||
tgui_say = ""
|
||||
mutable = ["", time.time()]
|
||||
def run_coorotine(mutable):
|
||||
async def get_result(mutable):
|
||||
async for response in run(prompt, max_token=20):
|
||||
print(response[len(mutable[0]):])
|
||||
mutable[0] = response
|
||||
if (time.time() - mutable[1]) > 3:
|
||||
model_name, addr_port = llm_kwargs['llm_model'].split('@')
|
||||
assert ':' in addr_port, "LLM_MODEL 格式不正确！" + llm_kwargs['llm_model']
|
||||
addr, port = addr_port.split(':')
|
||||
|
||||
|
||||
def run_coorotine(observe_window):
|
||||
async def get_result(observe_window):
|
||||
async for response in run(context=prompt, max_token=llm_kwargs['max_length'],
|
||||
temperature=llm_kwargs['temperature'],
|
||||
top_p=llm_kwargs['top_p'], addr=addr, port=port):
|
||||
print(response[len(observe_window[0]):])
|
||||
observe_window[0] = response
|
||||
if (time.time() - observe_window[1]) > 5:
|
||||
print('exit when no listener')
|
||||
break
|
||||
asyncio.run(get_result(mutable))
|
||||
thread_listen = threading.Thread(target=run_coorotine, args=(mutable,))
|
||||
asyncio.run(get_result(observe_window))
|
||||
thread_listen = threading.Thread(target=run_coorotine, args=(observe_window,))
|
||||
thread_listen.start()
|
||||
while thread_listen.is_alive():
|
||||
time.sleep(1)
|
||||
mutable[1] = time.time()
|
||||
tgui_say = mutable[0]
|
||||
return tgui_say
|
||||
return observe_window[0]
|
||||
|
||||
6
request_llm/requirements_chatglm.txt
Normal file
6
request_llm/requirements_chatglm.txt
Normal file
@ -0,0 +1,6 @@
|
||||
protobuf
|
||||
transformers==4.27.1
|
||||
cpm_kernels
|
||||
torch>=1.10
|
||||
mdtex2html
|
||||
sentencepiece
|
||||
Reference in New Issue
Block a user