功能: 实现外部API服务和设置对话框更新

- 在`PredictOption`结构体中添加`Stop`字段
- 添加一个新文件`external_api_service.go`，其中包含`NLP`接口的实现
- 添加一个名为`isExternalApi`的函数，用于检查文件是否为外部API
- 修改`modelLoad`，如果文件是外部API，则使用`ExternalApiService`
- 在`index.html`中的设置对话框中添加一个`external_api`字段

Author: wailovet

common.go

@@ -109,5 +109,6 @@ type PredictOption struct {
 	Tokens      int
 	MaxTokens   int
 	Threads     int
+	Stop        []string
 	StreamFn    func(outputText string) (stop bool)
 }

external_api_service.go

@@ -0,0 +1,147 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/parnurzeal/gorequest"
+	"github.com/tidwall/gjson"
+	"github.com/wailovet/nuwa"
+)
+
+type ExternalApiService struct {
+	Url string
+}
+
+// Free implements NLP
+func (e *ExternalApiService) Free() {
+
+}
+
+// IsReady implements NLP
+func (e *ExternalApiService) IsReady() bool {
+	return true
+}
+
+// ModelFile implements NLP
+func (e *ExternalApiService) ModelFile() string {
+	return e.Url
+}
+
+// Predict implements NLP
+func (e *ExternalApiService) Predict(p Prompts, his ChatHistory, opts *PredictOption) (string, error) {
+	text := fmt.Sprintln(p.Instruct)
+
+	if his == nil || len(his) == 0 {
+
+	} else {
+		if his[len(his)-1].Role == "assistant" {
+			if len(his)-1 > 0 {
+				his = his[:len(his)-1]
+			} else {
+				return "", fmt.Errorf("history is nil")
+			}
+		}
+
+		for _, v := range his {
+			if v.Role == "assistant" {
+				text += fmt.Sprintln(p.AssistantPrefix, v.Content)
+			} else {
+				text += fmt.Sprintln(p.UserPrefix, v.Content)
+			}
+		}
+		text += p.AssistantPrefix
+	}
+
+	log.Println(text)
+
+	req := gorequest.New()
+
+	if !strings.HasSuffix(e.Url, "/") {
+		e.Url += "/"
+	}
+	sendUrl := e.Url + "completion"
+
+	postJson := map[string]interface{}{
+		"prompt":      text,
+		"batch_size":  64,
+		"as_loop":     true,
+		"n_keep":      -1,
+		"interactive": true,
+		"stop":        []string{"\n### Human:"},
+	}
+	if opts != nil {
+		if opts.BatchSize > 0 {
+			postJson["batch_size"] = opts.BatchSize
+		}
+		if opts.Penalty > 0 {
+			postJson["penalty"] = opts.Penalty
+		}
+		if opts.Temperature > 0 {
+			postJson["temperature"] = opts.Temperature
+		}
+		if opts.TopP > 0 {
+			postJson["top_p"] = opts.TopP
+		}
+		if opts.TopK > 0 {
+			postJson["top_k"] = opts.TopK
+		}
+		if opts.Tokens > 0 {
+			postJson["n_predict"] = opts.Tokens
+		}
+		if opts.Threads > 0 {
+			postJson["threads"] = opts.Threads
+		}
+		if opts.Stop != nil {
+			postJson["stop"] = opts.Stop
+		}
+	} else {
+		return "", fmt.Errorf("opts is nil")
+	}
+
+	req.Post(sendUrl).Send(nuwa.Helper().JsonEncode(postJson)).End()
+
+	nextTokenUrl := e.Url + "next-token"
+
+	message := ""
+	isStop := false
+	for {
+		var errs []error
+		var result string
+		if isStop {
+			_, result, errs = req.Get(nextTokenUrl + "?stop=true").End()
+		} else {
+
+			_, result, errs = req.Get(nextTokenUrl).End()
+		}
+		if errs != nil {
+			return "", errs[0]
+		}
+		// message += result.data.content;
+		// if (result.data.stop) {
+		//     console.log("Completed");
+		//     // make sure to add the completion to the prompt.
+		//     prompt += `### Assistant: ${message}`;
+		//     break;
+		// }
+		message += gjson.Get(result, "content").String()
+		if gjson.Get(result, "stop").Bool() {
+			break
+		}
+
+		log.Println("web.result:", result)
+
+		if !isStop {
+			isStop = opts.StreamFn(message)
+		}
+	}
+
+	return message, nil
+}
+
+// StartUp implements NLP
+func (e *ExternalApiService) StartUp(modelfile string) error {
+	e.Url = modelfile
+	return nil
+}

main.go

@@ -39,28 +39,41 @@ var wsConnMapLock sync.RWMutex               //ws连接池锁
 
 var adminHome, _ = os.UserHomeDir() //用户目录
 
+func isExternalApi(filename string) bool {
+	if strings.HasPrefix(filename, "http://") || strings.HasPrefix(filename, "https://") {
+		return true
+	}
+	return false
+}
+
 func modelLoad(filename string) error {
 	if lm != nil {
 		lm.Free()
 	}
 
-	_modelType := modelType(filename)
+	if isExternalApi(filename) {
+		lm = &ExternalApiService{}
+		lm.StartUp(filename)
+	} else {
+		_modelType := modelType(filename)
 
-	log.Println("modelType:", _modelType)
+		log.Println("modelType:", _modelType)
 
-	switch _modelType {
-	case "llama":
-		lm = &Llama{}
-	case "rwkv":
-		lm = &RWKV{}
-	default:
-		lm = &Llama{}
-	}
+		switch _modelType {
+		case "llama":
+			lm = &Llama{}
+		case "rwkv":
+			lm = &RWKV{}
+		default:
+			lm = &Llama{}
+		}
 
-	err := lm.StartUp(filename)
-	if err != nil {
-		return err
+		err := lm.StartUp(filename)
+		if err != nil {
+			return err
+		}
 	}
+
 	return nil
 }
 
@@ -88,11 +101,16 @@ func serviceStartUp() {
 
 		nuwa.Http().HandleFunc("/model/reload", func(ctx nuwa.HttpContext) {
 			basePath := ctx.REQUEST["base_path"]
+			// external_api
+			filename := ctx.REQUEST["external_api"]
+
 			if basePath == "" {
 				basePath, _ = nuwa.Helper().GetCurrentPath()
 			}
-			filename := ctx.ParamRequired("filename")
-			filename = filepath.Join(basePath, filename)
+			if filename == "" {
+				filename = ctx.REQUEST["filename"]
+				filename = filepath.Join(basePath, filename)
+			}
 			err := modelLoad(filename)
 			ctx.CheckErrDisplayByError(err)
 			ctx.DisplayByData(filename)
@@ -254,6 +272,7 @@ func serviceStartUp() {
 				TopP:        topP,
 				Tokens:      tokens,
 				Threads:     threads,
+				Stop:        []string{stop_words},
 				StreamFn: func(outputText string) (stop bool) {
 					if strings.HasSuffix(outputText, stop_words) {
 						return true

src/index.html

@@ -504,8 +504,16 @@
             </div>
 
             <el-dialog title="设置" :visible="active_screen=='setting'" :show-close="false" center width="800px">
-                <div style="height: 400px;width: 740px;">
+                <div style="height: 500px;width: 740px;">
                     <el-form ref="form" label-width="100px" :inline="true">
+                        <el-form-item label="外部api地址">
+                            <el-input size="mini" v-model="external_api" placeholder="llama.cpp服务端外部api地址"
+                                style="width: 600px;"></el-input>
+                        </el-form-item>
+                    </el-form>
+
+
+                    <el-form ref="form" label-width="100px" :inline="true" v-if="external_api==''">
                         <el-form-item label="模型文件夹">
                             <el-input size="mini" v-model="nlp_model_base_path" placeholder="留空为当前执行程序所在文件夹"
                                 style="width: 200px;"></el-input>
@@ -517,7 +525,6 @@
                                 </el-option>
                             </el-select>
                         </el-form-item>
-
                     </el-form>
 
 
@@ -565,8 +572,7 @@
                         <el-form-item label="batch">
                             <el-input-number size="mini" v-model="predict_config.batch" :step="1">
                             </el-input-number>
-                            <el-tooltip class="item" effect="dark" content="CUDA必须要大于32才会生效"
-                                placement="top-start">
+                            <el-tooltip class="item" effect="dark" content="CUDA必须要大于32才会生效" placement="top-start">
                                 <svg t="1678602279133" class="icon" viewBox="0 0 1024 1024" version="1.1"
                                     xmlns="http://www.w3.org/2000/svg" p-id="4047" width="12" height="12">
                                     <path

src/index.js

@@ -87,6 +87,7 @@ function initVue() {
         session_id: "",
         load_lock: false,
         nlp_model: "",
+        external_api: "",
         nlp_model_base_path: "",
         nlp_model_list: [],
         history: [],
@@ -138,6 +139,9 @@ function initVue() {
       nlp_model: function (val) {
         storageSetValue("config.nlp_model", val);
       },
+      external_api: function (val) {
+        storageSetValue("config.external_api", val);
+      },
       predict_config: {
         handler: function (val) {
           storageSetValue("config.predict_config", JSON.stringify(val));
@@ -324,6 +328,7 @@ async function loadPage() {
 async function loadConfig() {
   app.nlp_model_base_path = await storageGetValue("config.nlp_model_base_path");
   app.nlp_model = await storageGetValue("config.nlp_model");
+  app.external_api = await storageGetValue("config.external_api");
   let predict_config = await storageGetValue("config.predict_config");
   if (predict_config) {
     app.predict_config = JSON.parse(predict_config);
@@ -333,6 +338,7 @@ async function loadConfig() {
 async function loadModel() {
   app.load_lock = true;
   let res = await axios.post(`${host}/model/reload`, {
+    external_api: app.external_api,
     base_path: app.nlp_model_base_path,
     filename: app.nlp_model,
   });