feat(task): replace built-in task adaptors with a sandboxed JS plugin system (#7076)

2026-09-11 14:41:21 +00:00 · 2026-08-29 18:51:57 +08:00
parent 7037ac15bd
commit eb48396d5f
336 changed files with 52333 additions and 6369 deletions
@@ -1,6 +1,7 @@
 package system_setting

 var ServerAddress = "http://localhost:3000"
+var TaskPublicAddress = ""
 var WorkerUrl = ""
 var WorkerValidKey = ""
 var WorkerAllowHttpImageRequestEnabled = false
@@ -0,0 +1,56 @@
+package system_setting
+
+import "github.com/QuantumNous/new-api/common"
+
+const (
+	DefaultTaskArtifactInvalidRateLimitPerMinute = 60
+	DefaultTaskArtifactGlobalConcurrency         = 128
+	DefaultTaskArtifactIPConcurrency             = 64
+	DefaultTaskArtifactObjectConcurrency         = 16
+)
+
+const (
+	TaskArtifactInvalidRateLimitEnv = "TASK_ARTIFACT_INVALID_RATE_LIMIT_PER_MINUTE"
+	TaskArtifactGlobalLimitEnv      = "TASK_ARTIFACT_GLOBAL_CONCURRENCY"
+	TaskArtifactIPLimitEnv          = "TASK_ARTIFACT_IP_CONCURRENCY"
+	TaskArtifactObjectLimitEnv      = "TASK_ARTIFACT_OBJECT_CONCURRENCY"
+)
+
+type TaskArtifactAccessLimits struct {
+	InvalidRatePerMinute int
+	GlobalConcurrency    int
+	IPConcurrency        int
+	ObjectConcurrency    int
+}
+
+func positiveTaskArtifactLimit(env string, defaultValue int) int {
+	value := common.GetEnvOrDefault(env, defaultValue)
+	if value <= 0 {
+		common.SysError(env + " must be positive; using default")
+		return defaultValue
+	}
+	return value
+}
+
+// LoadTaskArtifactAccessLimits reads startup configuration. Invalid and
+// non-positive values safely fall back to the documented defaults.
+func LoadTaskArtifactAccessLimits() TaskArtifactAccessLimits {
+	return TaskArtifactAccessLimits{
+		InvalidRatePerMinute: positiveTaskArtifactLimit(
+			TaskArtifactInvalidRateLimitEnv,
+			DefaultTaskArtifactInvalidRateLimitPerMinute,
+		),
+		GlobalConcurrency: positiveTaskArtifactLimit(
+			TaskArtifactGlobalLimitEnv,
+			DefaultTaskArtifactGlobalConcurrency,
+		),
+		IPConcurrency: positiveTaskArtifactLimit(
+			TaskArtifactIPLimitEnv,
+			DefaultTaskArtifactIPConcurrency,
+		),
+		ObjectConcurrency: positiveTaskArtifactLimit(
+			TaskArtifactObjectLimitEnv,
+			DefaultTaskArtifactObjectConcurrency,
+		),
+	}
+}
@@ -0,0 +1,165 @@
+package system_setting
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"regexp"
+	"strings"
+	"unicode"
+
+	"github.com/QuantumNous/new-api/common"
+)
+
+const (
+	TaskArtifactStoreModeUpstream = "upstream"
+	TaskArtifactStoreModeS3       = "s3"
+
+	DefaultTaskArtifactStorePresignTTLSeconds = 900
+	MaxTaskArtifactStorePresignTTLSeconds     = 7 * 24 * 60 * 60
+)
+
+const (
+	TaskArtifactStoreModeEnv         = "TASK_ARTIFACT_STORE_MODE"
+	TaskArtifactStoreS3EndpointEnv   = "TASK_ARTIFACT_STORE_S3_ENDPOINT"
+	TaskArtifactStoreS3BucketEnv     = "TASK_ARTIFACT_STORE_S3_BUCKET"
+	TaskArtifactStoreS3RegionEnv     = "TASK_ARTIFACT_STORE_S3_REGION"
+	TaskArtifactStoreS3AccessKeyEnv  = "TASK_ARTIFACT_STORE_S3_ACCESS_KEY"
+	TaskArtifactStoreS3SecretKeyEnv  = "TASK_ARTIFACT_STORE_S3_SECRET_KEY"
+	TaskArtifactStoreS3PrefixEnv     = "TASK_ARTIFACT_STORE_S3_PREFIX"
+	TaskArtifactStoreS3PresignTTLEnv = "TASK_ARTIFACT_STORE_S3_PRESIGN_TTL"
+)
+
+var (
+	taskArtifactStoreBucketPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$`)
+	taskArtifactStoreRegionPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`)
+)
+
+// TaskArtifactStoreConfig reserves the configuration contract for a future S3
+// implementation. The current release always falls back to upstream proxying.
+type TaskArtifactStoreConfig struct {
+	Mode                string
+	S3Endpoint          string
+	S3Bucket            string
+	S3Region            string
+	S3AccessKey         string
+	S3SecretKey         string
+	S3Prefix            string
+	S3PresignTTLSeconds int
+}
+
+// LoadTaskArtifactStoreConfig reads and validates startup-only configuration.
+// S3 mode is deliberately disabled until a storage implementation is shipped.
+func LoadTaskArtifactStoreConfig() TaskArtifactStoreConfig {
+	config := TaskArtifactStoreConfig{
+		Mode:                common.GetEnvOrDefaultString(TaskArtifactStoreModeEnv, TaskArtifactStoreModeUpstream),
+		S3Endpoint:          common.GetEnvOrDefaultString(TaskArtifactStoreS3EndpointEnv, ""),
+		S3Bucket:            common.GetEnvOrDefaultString(TaskArtifactStoreS3BucketEnv, ""),
+		S3Region:            common.GetEnvOrDefaultString(TaskArtifactStoreS3RegionEnv, ""),
+		S3AccessKey:         common.GetEnvOrDefaultString(TaskArtifactStoreS3AccessKeyEnv, ""),
+		S3SecretKey:         common.GetEnvOrDefaultString(TaskArtifactStoreS3SecretKeyEnv, ""),
+		S3Prefix:            common.GetEnvOrDefaultString(TaskArtifactStoreS3PrefixEnv, ""),
+		S3PresignTTLSeconds: common.GetEnvOrDefault(TaskArtifactStoreS3PresignTTLEnv, DefaultTaskArtifactStorePresignTTLSeconds),
+	}
+	if err := ValidateTaskArtifactStoreConfig(config); err != nil {
+		common.SysError("invalid task artifact store configuration: " + err.Error() + "; using upstream mode")
+		config.Mode = TaskArtifactStoreModeUpstream
+		return config
+	}
+	if config.Mode == TaskArtifactStoreModeS3 {
+		common.SysError("task artifact S3 storage is not implemented; using upstream mode")
+		config.Mode = TaskArtifactStoreModeUpstream
+	}
+	return config
+}
+
+// ValidateTaskArtifactStoreConfig performs syntax checks only. It never
+// resolves hosts, contacts an endpoint, or verifies credentials.
+func ValidateTaskArtifactStoreConfig(config TaskArtifactStoreConfig) error {
+	if config.Mode != TaskArtifactStoreModeUpstream && config.Mode != TaskArtifactStoreModeS3 {
+		return fmt.Errorf("unsupported mode %q", config.Mode)
+	}
+	if config.S3PresignTTLSeconds <= 0 || config.S3PresignTTLSeconds > MaxTaskArtifactStorePresignTTLSeconds {
+		return fmt.Errorf("S3 presign TTL must be between 1 and %d seconds", MaxTaskArtifactStorePresignTTLSeconds)
+	}
+
+	requireS3Fields := config.Mode == TaskArtifactStoreModeS3
+	if requireS3Fields && config.S3Endpoint == "" {
+		return errors.New("S3 endpoint is required")
+	}
+	if config.S3Endpoint != "" {
+		if config.S3Endpoint != strings.TrimSpace(config.S3Endpoint) {
+			return errors.New("S3 endpoint must not contain surrounding whitespace")
+		}
+		endpoint, err := url.Parse(config.S3Endpoint)
+		if err != nil || endpoint == nil || endpoint.Host == "" || endpoint.User != nil || endpoint.Opaque != "" {
+			return errors.New("S3 endpoint must be an absolute URL without userinfo")
+		}
+		if endpoint.Scheme != "http" && endpoint.Scheme != "https" {
+			return errors.New("S3 endpoint must use http or https")
+		}
+		if endpoint.RawQuery != "" || endpoint.ForceQuery || endpoint.Fragment != "" {
+			return errors.New("S3 endpoint must not contain a query or fragment")
+		}
+	}
+
+	if requireS3Fields && config.S3Bucket == "" {
+		return errors.New("S3 bucket is required")
+	}
+	if config.S3Bucket != "" {
+		if !taskArtifactStoreBucketPattern.MatchString(config.S3Bucket) ||
+			strings.Contains(config.S3Bucket, "..") || net.ParseIP(config.S3Bucket) != nil {
+			return errors.New("S3 bucket syntax is invalid")
+		}
+	}
+
+	if requireS3Fields && config.S3Region == "" {
+		return errors.New("S3 region is required")
+	}
+	if config.S3Region != "" && !taskArtifactStoreRegionPattern.MatchString(config.S3Region) {
+		return errors.New("S3 region syntax is invalid")
+	}
+	if err := validateTaskArtifactStoreCredential("access key", config.S3AccessKey, 256, requireS3Fields); err != nil {
+		return err
+	}
+	if err := validateTaskArtifactStoreCredential("secret key", config.S3SecretKey, 1024, requireS3Fields); err != nil {
+		return err
+	}
+
+	if config.S3Prefix != "" {
+		if config.S3Prefix != strings.TrimSpace(config.S3Prefix) || len(config.S3Prefix) > 512 ||
+			strings.HasPrefix(config.S3Prefix, "/") || strings.Contains(config.S3Prefix, "\\") {
+			return errors.New("S3 prefix syntax is invalid")
+		}
+		for _, part := range strings.Split(config.S3Prefix, "/") {
+			if part == "." || part == ".." {
+				return errors.New("S3 prefix must not contain dot segments")
+			}
+		}
+		for _, character := range config.S3Prefix {
+			if unicode.IsControl(character) {
+				return errors.New("S3 prefix must not contain control characters")
+			}
+		}
+	}
+	return nil
+}
+
+func validateTaskArtifactStoreCredential(name, value string, maxLength int, required bool) error {
+	if required && value == "" {
+		return fmt.Errorf("S3 %s is required", name)
+	}
+	if value == "" {
+		return nil
+	}
+	if value != strings.TrimSpace(value) || len(value) > maxLength {
+		return fmt.Errorf("S3 %s syntax is invalid", name)
+	}
+	for _, character := range value {
+		if unicode.IsControl(character) {
+			return fmt.Errorf("S3 %s syntax is invalid", name)
+		}
+	}
+	return nil
+}
@@ -0,0 +1,76 @@
+package system_setting
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestValidateTaskArtifactStoreConfig(t *testing.T) {
+	valid := TaskArtifactStoreConfig{
+		Mode:                TaskArtifactStoreModeS3,
+		S3Endpoint:          "https://objects.example.com/storage",
+		S3Bucket:            "task-artifacts",
+		S3Region:            "us-east-1",
+		S3AccessKey:         "access-key",
+		S3SecretKey:         "secret-key",
+		S3Prefix:            "tasks/v1/",
+		S3PresignTTLSeconds: 900,
+	}
+	require.NoError(t, ValidateTaskArtifactStoreConfig(valid))
+	require.NoError(t, ValidateTaskArtifactStoreConfig(TaskArtifactStoreConfig{
+		Mode:                TaskArtifactStoreModeUpstream,
+		S3PresignTTLSeconds: DefaultTaskArtifactStorePresignTTLSeconds,
+	}))
+
+	tests := []struct {
+		name   string
+		mutate func(*TaskArtifactStoreConfig)
+		match  string
+	}{
+		{name: "mode", mutate: func(config *TaskArtifactStoreConfig) { config.Mode = "filesystem" }, match: "unsupported mode"},
+		{name: "endpoint scheme", mutate: func(config *TaskArtifactStoreConfig) { config.S3Endpoint = "ftp://objects.example.com" }, match: "http or https"},
+		{name: "endpoint credentials", mutate: func(config *TaskArtifactStoreConfig) { config.S3Endpoint = "https://user:pass@objects.example.com" }, match: "without userinfo"},
+		{name: "endpoint query", mutate: func(config *TaskArtifactStoreConfig) { config.S3Endpoint = "https://objects.example.com?token=secret" }, match: "query or fragment"},
+		{name: "bucket", mutate: func(config *TaskArtifactStoreConfig) { config.S3Bucket = "Invalid_Bucket" }, match: "bucket syntax"},
+		{name: "IP bucket", mutate: func(config *TaskArtifactStoreConfig) { config.S3Bucket = "192.168.1.1" }, match: "bucket syntax"},
+		{name: "region", mutate: func(config *TaskArtifactStoreConfig) { config.S3Region = "bad region" }, match: "region syntax"},
+		{name: "access key", mutate: func(config *TaskArtifactStoreConfig) { config.S3AccessKey = " access-key" }, match: "access key syntax"},
+		{name: "secret key", mutate: func(config *TaskArtifactStoreConfig) { config.S3SecretKey = "secret\nkey" }, match: "secret key syntax"},
+		{name: "prefix root", mutate: func(config *TaskArtifactStoreConfig) { config.S3Prefix = "/tasks" }, match: "prefix syntax"},
+		{name: "prefix traversal", mutate: func(config *TaskArtifactStoreConfig) { config.S3Prefix = "tasks/../private" }, match: "dot segments"},
+		{name: "TTL zero", mutate: func(config *TaskArtifactStoreConfig) { config.S3PresignTTLSeconds = 0 }, match: "presign TTL"},
+		{name: "TTL too long", mutate: func(config *TaskArtifactStoreConfig) {
+			config.S3PresignTTLSeconds = MaxTaskArtifactStorePresignTTLSeconds + 1
+		}, match: "presign TTL"},
+	}
+	for _, testCase := range tests {
+		t.Run(testCase.name, func(t *testing.T) {
+			config := valid
+			testCase.mutate(&config)
+			assert.ErrorContains(t, ValidateTaskArtifactStoreConfig(config), testCase.match)
+		})
+	}
+}
+
+func TestLoadTaskArtifactStoreConfigFallsBackToUpstream(t *testing.T) {
+	t.Setenv(TaskArtifactStoreModeEnv, "filesystem")
+	t.Setenv(TaskArtifactStoreS3PresignTTLEnv, "900")
+	config := LoadTaskArtifactStoreConfig()
+	assert.Equal(t, TaskArtifactStoreModeUpstream, config.Mode)
+
+	t.Setenv(TaskArtifactStoreModeEnv, TaskArtifactStoreModeS3)
+	t.Setenv(TaskArtifactStoreS3EndpointEnv, "https://objects.example.com")
+	t.Setenv(TaskArtifactStoreS3BucketEnv, "task-artifacts")
+	t.Setenv(TaskArtifactStoreS3RegionEnv, "us-east-1")
+	t.Setenv(TaskArtifactStoreS3AccessKeyEnv, "access-key")
+	t.Setenv(TaskArtifactStoreS3SecretKeyEnv, "secret-key")
+	t.Setenv(TaskArtifactStoreS3PrefixEnv, "tasks/v1")
+	t.Setenv(TaskArtifactStoreS3PresignTTLEnv, "600")
+	config = LoadTaskArtifactStoreConfig()
+
+	assert.Equal(t, TaskArtifactStoreModeUpstream, config.Mode)
+	assert.Equal(t, "https://objects.example.com", config.S3Endpoint)
+	assert.Equal(t, 600, config.S3PresignTTLSeconds)
+}
@@ -0,0 +1,33 @@
+package system_setting
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadTaskArtifactAccessLimitsUsesPositiveEnvironmentValues(t *testing.T) {
+	t.Setenv(TaskArtifactInvalidRateLimitEnv, "17")
+	t.Setenv(TaskArtifactGlobalLimitEnv, "23")
+	t.Setenv(TaskArtifactIPLimitEnv, "11")
+	t.Setenv(TaskArtifactObjectLimitEnv, "7")
+
+	limits := LoadTaskArtifactAccessLimits()
+	assert.Equal(t, 17, limits.InvalidRatePerMinute)
+	assert.Equal(t, 23, limits.GlobalConcurrency)
+	assert.Equal(t, 11, limits.IPConcurrency)
+	assert.Equal(t, 7, limits.ObjectConcurrency)
+}
+
+func TestLoadTaskArtifactAccessLimitsFallsBackForInvalidValues(t *testing.T) {
+	t.Setenv(TaskArtifactInvalidRateLimitEnv, "0")
+	t.Setenv(TaskArtifactGlobalLimitEnv, "-1")
+	t.Setenv(TaskArtifactIPLimitEnv, "invalid")
+	t.Setenv(TaskArtifactObjectLimitEnv, "")
+
+	limits := LoadTaskArtifactAccessLimits()
+	assert.Equal(t, DefaultTaskArtifactInvalidRateLimitPerMinute, limits.InvalidRatePerMinute)
+	assert.Equal(t, DefaultTaskArtifactGlobalConcurrency, limits.GlobalConcurrency)
+	assert.Equal(t, DefaultTaskArtifactIPConcurrency, limits.IPConcurrency)
+	assert.Equal(t, DefaultTaskArtifactObjectConcurrency, limits.ObjectConcurrency)
+}