主要使用 fastchat 进行构建，其仓库地址：https://github.com/lm-sys/FastChat
# 构建虚拟环境
conda create --name testapi python==3.10
# 进入虚拟环境
conda activate testapi
# 更新一下
python -m pip install --upgrade pip
# 安装库
pip3 install "fschat[model_worker,webui]"
# 输入以下命令
# 启动控制器服务
python3 -m fastchat.serve.controller --host 127.0.0.1
# 启动模型服务
python3 -m fastchat.serve.model_worker --model-path ./Llama-2-70b-chat-hf --num-gpus 7 --host 127.0.0.1 --worker-address http://127.0.0.1:21002 --controller-address http://127.0.0.1:21001 # 请注意输出中的模型名称，用于调用
# 启动API服务
python3 -m fastchat.serve.openai_api_server --host 127.0.0.1 --port 8000
# 启动web服务（未尝试是否可以）服务默认端口是 7860，可以通过--port参数来修改端口，还可以通过添加--share参数来开启 Gradio 的共享模式，这样就可以通过外网访问 WebUI 服务了
python -m fastchat.serve.gradio_web_server --host 0.0.0.0
# 使用openai 进行调用
# fastchat 官方文档：https://github.com/lm-sys/FastChat/blob/main/docs/openai_api.md
import openaiopenai.api_key = "EMPTY"
openai.base_url = "http://localhost:8000/v1/"model = "Llama-2-70b-chat-hf"
prompt = "Once upon a time"# create a completion
completion = openai.completions.create(model=model, prompt=prompt, max_tokens=64)
# print the completion
print(prompt + completion.choices[0].text)# create a chat completion
completion = openai.chat.completions.create(model=model,messages=[{"role": "user", "content": "Hello! What is your name?"}]
)
# print the completion
print(completion.choices[0].message.content)## 如果输出你觉得没有完成，请再次访问
# completion = openai.chat.completions.create(
# model=model,
# messages=[{"role": "user", "content": "Hello! What is your name?"},
# {"role": "assistant", "content": completion.choices[0].message.content},
# {"role": "user", "content": "Continue."}]
# )
# # print the completion
# print(completion.choices[0].message.content)另一种方法
安装
pip install uvicorn
pip install fastapi
pip install pydantic
pip install torch
pip install transformers
# 服务端使用文件
import uvicorn
from fastapi import FastAPI
from pydantic import BaseModel
import uvicorn, json, datetime
import torch
import os
from transformers import AutoModelForCausalLM, AutoTokenizer
from transformers.generation.utils import GenerationConfig
os.environ['CUDA_VISIBLE_DEVICES'] = "5,6,7"app = FastAPI()class Query(BaseModel):text: strpath = "/workdir/model/baichuan13b_chat/"
tokenizer = AutoTokenizer.from_pretrained(path, use_fast=False, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(path, device_map="auto", torch_dtype=torch.float16, trust_remote_code=True)
model.generation_config = GenerationConfig.from_pretrained(path)@app.post("/chat/")
async def chat(query: Query):input_ids = tokenizer([query.text]).input_idsoutput_ids = model.generate(torch.as_tensor(input_ids).cuda(),do_sample=False,temperature=0.1,repetition_penalty=1,max_new_tokens=1024)output_ids = output_ids[0][len(input_ids[0]):]outputs = tokenizer.decode(output_ids, skip_special_tokens=True, spaces_between_special_tokens=False)return {"result": outputs}if __name__ == "__main__":uvicorn.run(app, host="0.0.0.0", port=6667)
# 访问调用
import requestsurl = "http://0.0.0.0:6667/chat/"
query = {"text": "你好，请做一段自我介绍。"}response = requests.post(url, json=query)if response.status_code == 200:result = response.json()print("BOT:", result["result"])
else:print("Error:", response.status_code, response.text)