解决冲突，合并master

2023-04-28 09:58:16 +08:00
parent 4f7ff3bb42 a219512045
commit 26c20ed2f5
19 changed files with 1113 additions and 254 deletions
--- a/crazy_functions/crazy_utils.py
+++ b/crazy_functions/crazy_utils.py
@ -1,5 +1,4 @@
-import traceback
-from toolbox import update_ui, get_conf
+from toolbox import update_ui, get_conf, trimmed_format_exc

 def input_clipping(inputs, history, max_token_limit):
    import numpy as np
@ -94,12 +93,12 @@ def request_gpt_model_in_new_thread_with_ui_alive(
                    continue # 返回重试
                else:
                    # 【选择放弃】
-                    tb_str = '```\n' + traceback.format_exc() + '```'
+                    tb_str = '```\n' + trimmed_format_exc() + '```'
                    mutable[0] += f"[Local Message] 警告，在执行过程中遭遇问题, Traceback：\n\n{tb_str}\n\n"
                    return mutable[0] # 放弃
            except:
                # 【第三种情况】：其他错误：重试几次
-                tb_str = '```\n' + traceback.format_exc() + '```'
+                tb_str = '```\n' + trimmed_format_exc() + '```'
                print(tb_str)
                mutable[0] += f"[Local Message] 警告，在执行过程中遭遇问题, Traceback：\n\n{tb_str}\n\n"
                if retry_op > 0:
@ -173,7 +172,7 @@ def request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency(
    if max_workers == -1: # 读取配置文件
        try: max_workers, = get_conf('DEFAULT_WORKER_NUM')
        except: max_workers = 8
-        if max_workers <= 0 or max_workers >= 20: max_workers = 8
+        if max_workers <= 0: max_workers = 3
    # 屏蔽掉 chatglm的多线程，可能会导致严重卡顿
    if not (llm_kwargs['llm_model'].startswith('gpt-') or llm_kwargs['llm_model'].startswith('api2d-')):
        max_workers = 1
@ -220,14 +219,14 @@ def request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency(
                    continue # 返回重试
                else:
                    # 【选择放弃】
-                    tb_str = '```\n' + traceback.format_exc() + '```'
+                    tb_str = '```\n' + trimmed_format_exc() + '```'
                    gpt_say += f"[Local Message] 警告，线程{index}在执行过程中遭遇问题, Traceback：\n\n{tb_str}\n\n"
                    if len(mutable[index][0]) > 0: gpt_say += "此线程失败前收到的回答：\n\n" + mutable[index][0]
                    mutable[index][2] = "输入过长已放弃"
                    return gpt_say # 放弃
            except:
                # 【第三种情况】：其他错误
-                tb_str = '```\n' + traceback.format_exc() + '```'
+                tb_str = '```\n' + trimmed_format_exc() + '```'
                print(tb_str)
                gpt_say += f"[Local Message] 警告，线程{index}在执行过程中遭遇问题, Traceback：\n\n{tb_str}\n\n"
                if len(mutable[index][0]) > 0: gpt_say += "此线程失败前收到的回答：\n\n" + mutable[index][0]
--- a/crazy_functions/批量Markdown翻译.py
+++ b/crazy_functions/批量Markdown翻译.py
@ -84,7 +84,33 @@ def 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch
    yield from update_ui(chatbot=chatbot, history=history) # 刷新界面


+def get_files_from_everything(txt):
+    import glob, os

+    success = True
+    if txt.startswith('http'):
+        # 网络的远程文件
+        txt = txt.replace("https://github.com/", "https://raw.githubusercontent.com/")
+        txt = txt.replace("/blob/", "/")
+        import requests
+        from toolbox import get_conf
+        proxies, = get_conf('proxies')
+        r = requests.get(txt, proxies=proxies)
+        with open('./gpt_log/temp.md', 'wb+') as f: f.write(r.content)
+        project_folder = './gpt_log/'
+        file_manifest = ['./gpt_log/temp.md']
+    elif txt.endswith('.md'):
+        # 直接给定文件
+        file_manifest = [txt]
+        project_folder = os.path.dirname(txt)
+    elif os.path.exists(txt):
+        # 本地路径，递归搜索
+        project_folder = txt
+        file_manifest = [f for f in glob.glob(f'{project_folder}/**/*.md', recursive=True)]
+    else:
+        success = False
+
+    return success, file_manifest, project_folder


@CatchException
@ -98,6 +124,7 @@ def Markdown英译中(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_p
    # 尝试导入依赖，如果缺少依赖，则给出安装建议
    try:
        import tiktoken
+        import glob, os
    except:
        report_execption(chatbot, history,
                         a=f"解析项目: {txt}",
@ -105,19 +132,21 @@ def Markdown英译中(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_p
        yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
        return
    history = []    # 清空历史，以免输入溢出
-    import glob, os
-    if os.path.exists(txt):
-        project_folder = txt
-    else:
+
+    success, file_manifest, project_folder = get_files_from_everything(txt)
+
+    if not success:
+        # 什么都没有
        if txt == "": txt = '空空如也的输入栏'
        report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}")
        yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
        return
-    file_manifest = [f for f in glob.glob(f'{project_folder}/**/*.md', recursive=True)]
+
    if len(file_manifest) == 0:
        report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何.md文件: {txt}")
        yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
        return
+
    yield from 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='en->zh')


@ -135,6 +164,7 @@ def Markdown中译英(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_p
    # 尝试导入依赖，如果缺少依赖，则给出安装建议
    try:
        import tiktoken
+        import glob, os
    except:
        report_execption(chatbot, history,
                         a=f"解析项目: {txt}",
@ -142,18 +172,13 @@ def Markdown中译英(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_p
        yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
        return
    history = []    # 清空历史，以免输入溢出
-    import glob, os
-    if os.path.exists(txt):
-        project_folder = txt
-    else:
+    success, file_manifest, project_folder = get_files_from_everything(txt)
+    if not success:
+        # 什么都没有
        if txt == "": txt = '空空如也的输入栏'
        report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}")
        yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
        return
-    if txt.endswith('.md'):
-        file_manifest = [txt]
-    else:
-        file_manifest = [f for f in glob.glob(f'{project_folder}/**/*.md', recursive=True)]
    if len(file_manifest) == 0:
        report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何.md文件: {txt}")
        yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
--- a/crazy_functions/解析项目源代码.py
+++ b/crazy_functions/解析项目源代码.py
@ -1,5 +1,6 @@
 from toolbox import update_ui
 from toolbox import CatchException, report_execption, write_results_to_file
+from .crazy_utils import input_clipping

 def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt):
    import os, copy
@ -61,13 +62,15 @@ def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs,
        previous_iteration_files.extend([os.path.relpath(fp, project_folder) for index, fp in enumerate(this_iteration_file_manifest)])
        previous_iteration_files_string = ', '.join(previous_iteration_files)
        current_iteration_focus = ', '.join([os.path.relpath(fp, project_folder) for index, fp in enumerate(this_iteration_file_manifest)])
-        i_say = f'根据以上分析，对程序的整体功能和构架重新做出概括。然后用一张markdown表格整理每个文件的功能（包括{previous_iteration_files_string}）。'
+        i_say = f'用一张Markdown表格简要描述以下文件的功能：{previous_iteration_files_string}。根据以上分析，用一句话概括程序的整体功能。'
        inputs_show_user = f'根据以上分析，对程序的整体功能和构架重新做出概括，由于输入长度限制，可能需要分组处理，本组文件为 {current_iteration_focus} + 已经汇总的文件组。'
        this_iteration_history = copy.deepcopy(this_iteration_gpt_response_collection)
        this_iteration_history.append(last_iteration_result)
+        # 裁剪input
+        inputs, this_iteration_history_feed = input_clipping(inputs=i_say, history=this_iteration_history, max_token_limit=2560)
        result = yield from request_gpt_model_in_new_thread_with_ui_alive(
-            inputs=i_say, inputs_show_user=inputs_show_user, llm_kwargs=llm_kwargs, chatbot=chatbot,
-            history=this_iteration_history,   # 迭代之前的分析
+            inputs=inputs, inputs_show_user=inputs_show_user, llm_kwargs=llm_kwargs, chatbot=chatbot,
+            history=this_iteration_history_feed,   # 迭代之前的分析
            sys_prompt="你是一个程序架构分析师，正在分析一个项目的源代码。")
        report_part_2.extend([i_say, result])
        last_iteration_result = result
--- a/crazy_functions/谷歌检索小助手.py
+++ b/crazy_functions/谷歌检索小助手.py
@ -70,6 +70,7 @@ def 谷歌检索小助手(txt, llm_kwargs, plugin_kwargs, chatbot, history, syst
    # 尝试导入依赖，如果缺少依赖，则给出安装建议
    try:
        import arxiv
+        import math
        from bs4 import BeautifulSoup
    except:
        report_execption(chatbot, history, 
@ -80,25 +81,26 @@ def 谷歌检索小助手(txt, llm_kwargs, plugin_kwargs, chatbot, history, syst

    # 清空历史，以免输入溢出
    history = []
-
    meta_paper_info_list = yield from get_meta_information(txt, chatbot, history)
+    batchsize = 5
+    for batch in range(math.ceil(len(meta_paper_info_list)/batchsize)):
+        if len(meta_paper_info_list[:batchsize]) > 0:
+            i_say = "下面是一些学术文献的数据，提取出以下内容：" + \
+            "1、英文题目；2、中文题目翻译；3、作者；4、arxiv公开（is_paper_in_arxiv）；4、引用数量（cite）；5、中文摘要翻译。" + \
+            f"以下是信息源：{str(meta_paper_info_list[:batchsize])}" 

-    if len(meta_paper_info_list[:10]) > 0:
-        i_say = "下面是一些学术文献的数据，请从中提取出以下内容。" + \
-        "1、英文题目；2、中文题目翻译；3、作者；4、arxiv公开（is_paper_in_arxiv）；4、引用数量（cite）；5、中文摘要翻译。" + \
-        f"以下是信息源：{str(meta_paper_info_list[:10])}" 
+            inputs_show_user = f"请分析此页面中出现的所有文章：{txt}，这是第{batch+1}批"
+            gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(
+                inputs=i_say, inputs_show_user=inputs_show_user,
+                llm_kwargs=llm_kwargs, chatbot=chatbot, history=[],
+                sys_prompt="你是一个学术翻译，请从数据中提取信息。你必须使用Markdown表格。你必须逐个文献进行处理。"
+            )

-        inputs_show_user = f"请分析此页面中出现的所有文章：{txt}"
-        gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(
-            inputs=i_say, inputs_show_user=inputs_show_user, 
-            llm_kwargs=llm_kwargs, chatbot=chatbot, history=[], 
-            sys_prompt="你是一个学术翻译，请从数据中提取信息。你必须使用Markdown格式。你必须逐个文献进行处理。"
-        )
+            history.extend([ f"第{batch+1}批", gpt_say ])
+            meta_paper_info_list = meta_paper_info_list[batchsize:]

-        history.extend([ "第一批", gpt_say ])
-        meta_paper_info_list = meta_paper_info_list[10:]
-
-    chatbot.append(["状态？", "已经全部完成"])
+    chatbot.append(["状态？", 
+        "已经全部完成，您可以试试让AI写一个Related Works，例如您可以继续输入Write a \"Related Works\" section about \"你搜索的研究领域\" for me."])
    msg = '正常'
    yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面
    res = write_results_to_file(history)