mirror of
https://github.com/QuantumNous/new-api.git
synced 2026-09-12 15:21:09 +00:00
fix: adapt ClickHouse log LIKE filters
This commit is contained in:
@@ -101,6 +101,34 @@ func TestClickHouseLogOrder(t *testing.T) {
|
|||||||
assert.Equal(t, "logs.created_at desc, logs.request_id desc", clickHouseLogOrder("logs."))
|
assert.Equal(t, "logs.created_at desc, logs.request_id desc", clickHouseLogOrder("logs."))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBuildLogLikeConditionUsesStandardEscape(t *testing.T) {
|
||||||
|
originalLogDatabaseType := common.LogDatabaseType()
|
||||||
|
t.Cleanup(func() {
|
||||||
|
common.SetLogDatabaseType(originalLogDatabaseType)
|
||||||
|
})
|
||||||
|
common.SetLogDatabaseType(common.DatabaseTypeSQLite)
|
||||||
|
|
||||||
|
condition, pattern, err := buildLogLikeCondition("logs.model_name", "gpt_4%")
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "logs.model_name LIKE ? ESCAPE '!'", condition)
|
||||||
|
assert.Equal(t, "gpt!_4%", pattern)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildLogLikeConditionUsesClickHouseEscaping(t *testing.T) {
|
||||||
|
originalLogDatabaseType := common.LogDatabaseType()
|
||||||
|
t.Cleanup(func() {
|
||||||
|
common.SetLogDatabaseType(originalLogDatabaseType)
|
||||||
|
})
|
||||||
|
common.SetLogDatabaseType(common.DatabaseTypeClickHouse)
|
||||||
|
|
||||||
|
condition, pattern, err := buildLogLikeCondition("logs.model_name", `gpt_4\mini%`)
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "logs.model_name LIKE ?", condition)
|
||||||
|
assert.Equal(t, `gpt\_4\\mini%`, pattern)
|
||||||
|
}
|
||||||
|
|
||||||
func TestEnsureLogRequestId(t *testing.T) {
|
func TestEnsureLogRequestId(t *testing.T) {
|
||||||
empty := &Log{}
|
empty := &Log{}
|
||||||
ensureLogRequestId(empty)
|
ensureLogRequestId(empty)
|
||||||
|
|||||||
+28
-2
@@ -22,15 +22,41 @@ func applyExplicitLogTextFilter(tx *gorm.DB, column string, value string) (*gorm
|
|||||||
return tx, nil
|
return tx, nil
|
||||||
}
|
}
|
||||||
if strings.Contains(value, "%") {
|
if strings.Contains(value, "%") {
|
||||||
pattern, err := sanitizeLikePattern(value)
|
condition, pattern, err := buildLogLikeCondition(column, value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return tx.Where(column+" LIKE ? ESCAPE '!'", pattern), nil
|
return tx.Where(condition, pattern), nil
|
||||||
}
|
}
|
||||||
return tx.Where(column+" = ?", value), nil
|
return tx.Where(column+" = ?", value), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func buildLogLikeCondition(column string, value string) (string, string, error) {
|
||||||
|
if common.UsingLogDatabase(common.DatabaseTypeClickHouse) {
|
||||||
|
pattern, err := sanitizeClickHouseLikePattern(value)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
return column + " LIKE ?", pattern, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
pattern, err := sanitizeLikePattern(value)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
return column + " LIKE ? ESCAPE '!'", pattern, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sanitizeClickHouseLikePattern(input string) (string, error) {
|
||||||
|
input = strings.ReplaceAll(input, `\`, `\\`)
|
||||||
|
input = strings.ReplaceAll(input, `_`, `\_`)
|
||||||
|
|
||||||
|
if err := validateLikePattern(input); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return input, nil
|
||||||
|
}
|
||||||
|
|
||||||
type Log struct {
|
type Log struct {
|
||||||
Id int `json:"id" gorm:"index:idx_created_at_id,priority:2;index:idx_user_id_id,priority:2"`
|
Id int `json:"id" gorm:"index:idx_created_at_id,priority:2;index:idx_user_id_id,priority:2"`
|
||||||
UserId int `json:"user_id" gorm:"index;index:idx_user_id_id,priority:1"`
|
UserId int `json:"user_id" gorm:"index;index:idx_user_id_id,priority:1"`
|
||||||
|
|||||||
+25
-18
@@ -98,30 +98,37 @@ func sanitizeLikePattern(input string) (string, error) {
|
|||||||
input = strings.ReplaceAll(input, "!", "!!")
|
input = strings.ReplaceAll(input, "!", "!!")
|
||||||
input = strings.ReplaceAll(input, `_`, `!_`)
|
input = strings.ReplaceAll(input, `_`, `!_`)
|
||||||
|
|
||||||
// 2. 连续的 % 直接拒绝
|
if err := validateLikePattern(input); err != nil {
|
||||||
if strings.Contains(input, "%%") {
|
return "", err
|
||||||
return "", errors.New("搜索模式中不允许包含连续的 % 通配符")
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. 统计 % 数量,不得超过 2
|
|
||||||
count := strings.Count(input, "%")
|
|
||||||
if count > 2 {
|
|
||||||
return "", errors.New("搜索模式中最多允许包含 2 个 % 通配符")
|
|
||||||
}
|
|
||||||
|
|
||||||
// 4. 含 % 时,去掉 % 后关键词长度必须 >= 2
|
|
||||||
if count > 0 {
|
|
||||||
stripped := strings.ReplaceAll(input, "%", "")
|
|
||||||
if len(stripped) < 2 {
|
|
||||||
return "", errors.New("使用模糊搜索时,关键词长度至少为 2 个字符")
|
|
||||||
}
|
|
||||||
return input, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// 5. 无 % 时,精确全匹配
|
// 5. 无 % 时,精确全匹配
|
||||||
return input, nil
|
return input, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateLikePattern(input string) error {
|
||||||
|
// 1. 连续的 % 直接拒绝
|
||||||
|
if strings.Contains(input, "%%") {
|
||||||
|
return errors.New("搜索模式中不允许包含连续的 % 通配符")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. 统计 % 数量,不得超过 2
|
||||||
|
count := strings.Count(input, "%")
|
||||||
|
if count > 2 {
|
||||||
|
return errors.New("搜索模式中最多允许包含 2 个 % 通配符")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. 含 % 时,去掉 % 后关键词长度必须 >= 2
|
||||||
|
if count > 0 {
|
||||||
|
stripped := strings.ReplaceAll(input, "%", "")
|
||||||
|
if len(stripped) < 2 {
|
||||||
|
return errors.New("使用模糊搜索时,关键词长度至少为 2 个字符")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
const searchHardLimit = 100
|
const searchHardLimit = 100
|
||||||
|
|
||||||
func SearchUserTokens(userId int, keyword string, token string, offset int, limit int) (tokens []*Token, total int64, err error) {
|
func SearchUserTokens(userId int, keyword string, token string, offset int, limit int) (tokens []*Token, total int64, err error) {
|
||||||
|
|||||||
Reference in New Issue
Block a user