context is 모델 입력. context list 로관리된다.
message 는 messages.append({“role”: “user”, “content”: user_text}) 되는건데. 이제 여기서 user, system, assistant 가 있고, content 는 실제 내용 -> 추후 작성 더해야함.
uv python pin 3.12 uv add “transformers>=4.51” accelerate torch
from transformers import AutoModelForCausalLM, AutoTokenizer
model_name = "Qwen/Qwen3-1.7B"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(
model_name, torch_dtype="auto", device_map="auto"
)
messages = [] # context of the conversation, will be updated after each input and output
def chat(user_text, enable_thinking=False):
messages.append({"role": "user", "content": user_text})
text = tokenizer.apply_chat_template(
messages, tokenize=False,
add_generation_prompt=True,
enable_thinking=enable_thinking,
)
model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
n_in = model_inputs.input_ids.shape[1]
generated_ids = model.generate(**model_inputs, max_new_tokens=512)
output_ids = generated_ids[0][n_in:].tolist()
# Find the index of the first occurrence of the token 151668 in output_ids
try:
idx = len(output_ids) - output_ids[::-1].index(151668)
except ValueError:
idx = 0
reply = tokenizer.decode(output_ids[idx:], skip_special_tokens=True).strip("\n")
messages.append({"role": "assistant", "content": reply})
print(f"[input {n_in} tok → output {len(output_ids)} tok]")
print(f"USER: {user_text}")
print(f"ASSISTANT: {reply}\n")
return reply
chat("I am foxisdog, my favorite number is 7.") # first input
chat("What is my name and favorite number?. Answer only in name, number format") # second input
print(messages) # check messages list to see the context of the conversationuv run chat.py 2>&1 | tee log.txt
Loading weights: 0%| | 0/311 [00:00<?, ?it/s]
Loading weights: 1%| | 2/311 [00:00<00:19, 15.68it/s]
Loading weights: 10%|▉ | 30/311 [00:00<00:02, 118.25it/s]
Loading weights: 100%|██████████| 311/311 [00:00<00:00, 1028.22it/s]
[input 25 tok → output 126 tok]
USER: I am foxisdog, my favorite number is 7.
ASSISTANT: Hello, foxisdog! I'm glad to meet you and hear about your favorite number, 7. That's a great choice—it's often considered lucky, mysterious, and full of meaning in many cultures. If you're into numbers, you might enjoy exploring the symbolism of 7 in different contexts, like in the seven days of the week, the seven wonders of the world, or even in popular culture (like the seven dwarfs or the seven seas).
Would you like to discuss something specific about numbers, your favorite number, or maybe even a fun math puzzle? I'm here to help! 🐾
[input 175 tok → output 7 tok]
USER: What is my name and favorite number?. Answer only in name, number format
ASSISTANT: Foxisdog, 7
[{'role': 'user', 'content': 'I am foxisdog, my favorite number is 7.'}, {'role': 'assistant', 'content': "Hello, foxisdog! I'm glad to meet you and hear about your favorite number, 7. That's a great choice—it's often considered lucky, mysterious, and full of meaning in many cultures. If you're into numbers, you might enjoy exploring the symbolism of 7 in different contexts, like in the seven days of the week, the seven wonders of the world, or even in popular culture (like the seven dwarfs or the seven seas).\n\nWould you like to discuss something specific about numbers, your favorite number, or maybe even a fun math puzzle? I'm here to help! 🐾"}, {'role': 'user', 'content': 'What is my name and favorite number?. Answer only in name, number format'}, {'role': 'assistant', 'content': 'Foxisdog, 7'}]