ext/new-api
2
0
Fork 0
mirror of https://github.com/QuantumNous/new-api.git synced 2026-09-06 17:46:23 +00:00
Code Activity

feat: add persistent system task log cleanup progress

Browse Source
This commit is contained in:
CaIon
2026-06-22 19:16:53 +08:00
parent 6dc4030fdf
commit a162163b48
19 changed files with 984 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/log.go
+43 -34
View File
@@ -666,21 +666,25 @@ func SumUsedToken(logType int, startTimestamp int64, endTimestamp int64, modelNa
return token
}
func DeleteOldLog(ctx context.Context, targetTimestamp int64, limit int) (int64, error) {
func CountOldLog(ctx context.Context, targetTimestamp int64) (int64, error) {
var total int64
if err := LOG_DB.WithContext(ctx).Model(&Log{}).Where("created_at < ?", targetTimestamp).Count(&total).Error; err != nil {
return 0, err
}
return total, nil
}
func DeleteOldLogBatch(ctx context.Context, targetTimestamp int64, limit int) (int64, error) {
if limit <= 0 {
limit = 100
}
if nil != ctx.Err() {
return 0, ctx.Err()
}
if common.UsingLogDatabase(common.DatabaseTypeClickHouse) {
if limit <= 0 {
limit = 100
}
var total int64 = 0
for {
if nil != ctx.Err() {
return total, ctx.Err()
}
var batchCount int64
if err := LOG_DB.WithContext(ctx).Raw(`
var batchCount int64
if err := LOG_DB.WithContext(ctx).Raw(`
SELECT count() FROM (
SELECT created_at, request_id
FROM logs
@@ -688,13 +692,13 @@ SELECT count() FROM (
ORDER BY created_at ASC, request_id ASC
LIMIT ?
)`, targetTimestamp, limit).Scan(&batchCount).Error; err != nil {
return total, err
}
if batchCount == 0 {
break
}
return 0, err
}
if batchCount == 0 {
return 0, nil
}
if err := LOG_DB.WithContext(ctx).Exec(`
if err := LOG_DB.WithContext(ctx).Exec(`
ALTER TABLE logs DELETE WHERE (created_at, request_id) IN (
SELECT created_at, request_id
FROM logs
@@ -702,17 +706,22 @@ ALTER TABLE logs DELETE WHERE (created_at, request_id) IN (
ORDER BY created_at ASC, request_id ASC
LIMIT ?
) SETTINGS mutations_sync = 1`, targetTimestamp, limit).Error; err != nil {
return total, err
}
total += batchCount
if batchCount < int64(limit) {
break
}
return 0, err
}
return total, nil
return batchCount, nil
}
result := LOG_DB.WithContext(ctx).Where("created_at < ?", targetTimestamp).Limit(limit).Delete(&Log{})
if nil != result.Error {
return 0, result.Error
}
return result.RowsAffected, nil
}
func DeleteOldLog(ctx context.Context, targetTimestamp int64, limit int) (int64, error) {
if limit <= 0 {
limit = 100
}
var total int64 = 0
@@ -722,14 +731,14 @@ ALTER TABLE logs DELETE WHERE (created_at, request_id) IN (
return total, ctx.Err()
}
result := LOG_DB.Where("created_at < ?", targetTimestamp).Limit(limit).Delete(&Log{})
if nil != result.Error {
return total, result.Error
rowsAffected, err := DeleteOldLogBatch(ctx, targetTimestamp, limit)
if nil != err {
return total, err
}
total += result.RowsAffected
total += rowsAffected
if result.RowsAffected < int64(limit) {
if rowsAffected < int64(limit) {
break
}
}