feat alpha: support searxng

feat alpha: support searxng

Co-Authored-By: Minghan Zhang <[email protected]>

addition/web/call.go

@@ -10,10 +10,8 @@ import (
 
 type Hook func(message []globals.Message, token int) (string, error)
 
-func ChatWithWeb(message []globals.Message) []globals.Message {
-	data := utils.GetSegmentString(
-		SearchWebResult(message[len(message)-1].Content), 2048,
-	)
+func toWebSearchingMessage(message []globals.Message) []globals.Message {
+	data := GenerateSearchResult(message[len(message)-1].Content)
 
 	return utils.Insert(message, 0, globals.Message{
 		Role: globals.System,
@@ -24,19 +22,19 @@ func ChatWithWeb(message []globals.Message) []globals.Message {
 	})
 }
 
-func UsingWebSegment(instance *conversation.Conversation, restart bool) []globals.Message {
+func ToChatSearched(instance *conversation.Conversation, restart bool) []globals.Message {
 	segment := conversation.CopyMessage(instance.GetChatMessage(restart))
 
 	if instance.IsEnableWeb() {
-		segment = ChatWithWeb(segment)
+		segment = toWebSearchingMessage(segment)
 	}
 
 	return segment
 }
 
-func UsingWebNativeSegment(enable bool, message []globals.Message) []globals.Message {
+func ToSearched(enable bool, message []globals.Message) []globals.Message {
 	if enable {
-		return ChatWithWeb(message)
+		return toWebSearchingMessage(message)
 	} else {
 		return message
 	}

addition/web/search.go

@@ -4,16 +4,88 @@ import (
 	"chat/globals"
 	"chat/utils"
 	"fmt"
+	"net/url"
+	"strconv"
+	"strings"
 )
 
-func SearchWebResult(q string) string {
-	res, err := CallDuckDuckGoAPI(q)
+type SearXNGResponse struct {
+	Query           string `json:"query"`
+	NumberOfResults int    `json:"number_of_results"`
+	Results         []struct {
+		Url           string   `json:"url"`
+		Title         string   `json:"title"`
+		Content       string   `json:"content"`
+		PublishedDate *string  `json:"publishedDate,omitempty"`
+		Thumbnail     *string  `json:"thumbnail,omitempty"`
+		Engine        string   `json:"engine"`
+		ParsedUrl     []string `json:"parsed_url"`
+		Template      string   `json:"template"`
+		Engines       []string `json:"engines"`
+		Positions     []int    `json:"positions"`
+		Score         float64  `json:"score"`
+		Category      string   `json:"category"`
+		IframeSrc     string   `json:"iframe_src,omitempty"`
+	} `json:"results"`
+	Answers             []interface{} `json:"answers"`
+	Corrections         []interface{} `json:"corrections"`
+	Infoboxes           []interface{} `json:"infoboxes"`
+	Suggestions         []interface{} `json:"suggestions"`
+	UnresponsiveEngines [][]string    `json:"unresponsive_engines"`
+}
+
+func formatResponse(data *SearXNGResponse) string {
+	res := make([]string, 0)
+	for _, item := range data.Results {
+		if item.Content == "" || item.Url == "" || item.Title == "" {
+			continue
+		}
+
+		res = append(res, fmt.Sprintf("%s (%s): %s", item.Title, item.Url, item.Content))
+	}
+
+	return strings.Join(res, "\n")
+}
+
+func createURLParams(query string) string {
+	params := url.Values{}
+
+	params.Add("q", url.QueryEscape(query))
+	params.Add("format", "json")
+	params.Add("safesearch", strconv.Itoa(globals.SearchSafeSearch))
+	if len(globals.SearchEngines) > 0 {
+		params.Add("engines", globals.SearchEngines)
+	}
+	if len(globals.SearchImageProxy) > 0 {
+		params.Add("image_proxy", globals.SearchImageProxy)
+	}
+
+	return fmt.Sprintf("%s?%s", globals.SearchEndpoint, params.Encode())
+}
+
+func createSearXNGRequest(query string) (*SearXNGResponse, error) {
+	data, err := utils.Get(createURLParams(query), nil)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return utils.MapToRawStruct[SearXNGResponse](data)
+}
+
+func GenerateSearchResult(q string) string {
+	res, err := createSearXNGRequest(q)
 	if err != nil {
 		globals.Warn(fmt.Sprintf("[web] failed to get search result: %s (query: %s)", err.Error(), q))
 		return ""
 	}
 
 	content := formatResponse(res)
 	globals.Debug(fmt.Sprintf("[web] search result: %s (query: %s)", utils.Extract(content, 50, "..."), q))
+
+	if globals.SearchCrop {
+		globals.Debug(fmt.Sprintf("[web] crop search result length %d to %d max", len(content), globals.SearchCropLength))
+		return utils.Extract(content, globals.SearchCropLength, "...")
+	}
 	return content
 }

channel/system.go

@@ -63,9 +63,13 @@ type mailState struct {
 	WhiteList whiteList `json:"white_list" mapstructure:"whitelist"`
 }
 
-type searchState struct {
-	Endpoint string `json:"endpoint" mapstructure:"endpoint"`
-	Query    int    `json:"query" mapstructure:"query"`
+type SearchState struct {
+	Endpoint   string   `json:"endpoint" mapstructure:"endpoint"`
+	Crop       bool     `json:"crop" mapstructure:"crop"`
+	CropLen    int      `json:"crop_len" mapstructure:"croplen"`
+	Engines    []string `json:"engines" mapstructure:"engines"`
+	ImageProxy bool     `json:"image_proxy" mapstructure:"imageproxy"`
+	SafeSearch int      `json:"safe_search" mapstructure:"safesearch"`
 }
 
 type commonState struct {
@@ -113,6 +117,13 @@ func (c *SystemConfig) Load() {
 	if c.General.PWAManifest == "" {
 		c.General.PWAManifest = utils.ReadPWAManifest()
 	}
+
+	globals.SearchEndpoint = c.Search.Endpoint
+	globals.SearchCrop = c.Search.Crop
+	globals.SearchCropLength = c.GetSearchCropLength()
+	globals.SearchEngines = c.GetSearchEngines()
+	globals.SearchImageProxy = c.GetImageProxy()
+	globals.SearchSafeSearch = c.Search.SafeSearch
 }
 
 func (c *SystemConfig) SaveConfig() error {
@@ -220,27 +231,25 @@ func (c *SystemConfig) SendVerifyMail(email string, code string) error {
 	)
 }
 
-func (c *SystemConfig) GetSearchEndpoint() string {
-	if len(c.Search.Endpoint) == 0 {
-		return "https://duckduckgo-api.vercel.app"
+func (c *SystemConfig) GetSearchCropLength() int {
+	if c.Search.CropLen <= 0 {
+		return 1000
 	}
 
-	endpoint := c.Search.Endpoint
-	if strings.HasSuffix(endpoint, "/search") {
-		return endpoint[:len(endpoint)-7]
-	} else if strings.HasSuffix(endpoint, "/") {
-		return endpoint[:len(endpoint)-1]
-	}
+	return c.Search.CropLen
+}
 
-	return endpoint
+func (c *SystemConfig) GetSearchEngines() string {
+	return strings.Join(c.Search.Engines, ",")
 }
 
-func (c *SystemConfig) GetSearchQuery() int {
-	if c.Search.Query <= 0 {
-		return 5
+func (c *SystemConfig) GetImageProxy() string {
+	// return "True" or "False"
+	if c.Search.ImageProxy {
+		return "True"
 	}
 
-	return c.Search.Query
+	return "False"
 }
 
 func (c *SystemConfig) GetAppName() string {

globals/variables.go

@@ -1,9 +1,10 @@
 package globals
 
 import (
-	"github.com/gin-gonic/gin"
 	"net/url"
 	"strings"
+
+	"github.com/gin-gonic/gin"
 )
 
 const ChatMaxThread = 5
@@ -22,6 +23,13 @@ var AcceptImageStore bool
 var CloseRegistration bool
 var CloseRelay bool
 
+var SearchEndpoint string
+var SearchCrop bool
+var SearchCropLength int
+var SearchEngines string    // e.g. "google,bing"
+var SearchImageProxy string // e.g. "True", "False"
+var SearchSafeSearch int    // e.g. 0: None, 1: Moderation, 2: Strict
+
 func OriginIsAllowed(uri string) bool {
 	if len(AllowedOrigins) == 0 {
 		// if allowed origins is empty, allow all origins

manager/chat.go

@@ -197,7 +197,7 @@ func ChatHandler(conn *Connection, user *auth.User, instance *conversation.Conve
 	cache := conn.GetCache()
 
 	model := instance.GetModel()
-	segment := adapter.ClearMessages(model, web.UsingWebSegment(instance, restart))
+	segment := adapter.ClearMessages(model, web.ToChatSearched(instance, restart))
 
 	check, plan := auth.CanEnableModelWithSubscription(db, cache, user, model)
 	conn.Send(globals.ChatSegmentResponse{

manager/chat_completions.go

@@ -72,7 +72,7 @@ func ChatRelayAPI(c *gin.Context) {
 		suffix := strings.TrimPrefix(form.Model, "web-")
 
 		form.Model = suffix
-		messages = web.UsingWebNativeSegment(true, messages)
+		messages = web.ToSearched(true, messages)
 	}
 
 	if strings.HasSuffix(form.Model, "-official") {

manager/completions.go

@@ -1,16 +1,17 @@
 package manager
 
 import (
-	"chat/adapter/common"
+	adaptercommon "chat/adapter/common"
 	"chat/addition/web"
 	"chat/admin"
 	"chat/auth"
 	"chat/channel"
 	"chat/globals"
 	"chat/utils"
 	"fmt"
-	"github.com/gin-gonic/gin"
 	"runtime/debug"
+
+	"github.com/gin-gonic/gin"
 )
 
 func NativeChatHandler(c *gin.Context, user *auth.User, model string, message []globals.Message, enableWeb bool) (string, float32) {
@@ -23,7 +24,7 @@ func NativeChatHandler(c *gin.Context, user *auth.User, model string, message []
 		}
 	}()
 
-	segment := web.UsingWebNativeSegment(enableWeb, message)
+	segment := web.ToSearched(enableWeb, message)
 
 	db := utils.GetDBFromContext(c)
 	cache := utils.GetCacheFromContext(c)