From 1720dfa670f66bf20b173bbb5c9e5a2c6d905d26 Mon Sep 17 00:00:00 2001
From: herobrine19 <1909977364@qq.com>
Date: Tue, 23 May 2023 16:57:44 +0800
Subject: [PATCH] [FIX] Fix bug in finetune.py

---
 finetune.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/finetune.py b/finetune.py
index 4059fc7..ba019ff 100644
--- a/finetune.py
+++ b/finetune.py
@@ -123,7 +123,7 @@ def train(
     )
     tokenizer.padding_side = "left"  # Allow batched inference
 
-    def tokenize(prompt):
+    def tokenize(prompt, add_eos_token=True):
         # there's probably a way to do this with the tokenizer settings
         # but again, gotta move fast
         result = tokenizer(