new-api/controller/channel_inference_test.go

package controller

import (
	"context"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
	"testing"

	"github.com/QuantumNous/new-api/common"
	"github.com/QuantumNous/new-api/constant"
	"github.com/QuantumNous/new-api/model"
	"github.com/QuantumNous/new-api/service"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestVLLMStatusPartialFailureAndCredentials(t *testing.T) {
	service.InitHttpClient()
	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		assert.Equal(t, http.MethodGet, r.Method)
		assert.Equal(t, "Bearer test-vllm-key", r.Header.Get("Authorization"))
		assert.Equal(t, "status-panel", r.Header.Get("X-Source"))
		switch r.URL.Path {
		case "/prefix/health":
			w.WriteHeader(http.StatusOK)
		case "/prefix/version":
			_, _ = w.Write([]byte(`{"version":"0.25.2"}`))
		case "/prefix/v1/models":
			_, _ = w.Write([]byte(`{"data":[{"id":"served-model","root":"source/model","max_model_len":1048576}]}`))
		case "/prefix/metrics":
			http.Error(w, "test-vllm-key should not reach the browser", http.StatusForbidden)
		default:
			t.Errorf("unexpected path: %s", r.URL.Path)
			w.WriteHeader(http.StatusNotFound)
		}
	}))
	defer upstream.Close()
	base := upstream.URL + "/prefix/"
	override := `{"X-Source":"status-panel"}`
	channel := &model.Channel{Type: constant.ChannelTypeVLLM, BaseURL: &base, Key: "test-vllm-key", HeaderOverride: &override}
	status, err := fetchInferenceStatus(context.Background(), channel)
	require.NoError(t, err)
	assert.Equal(t, 200, status.Endpoints["/health"].Status)
	assert.Equal(t, "0.25.2", status.Version)
	require.Len(t, status.Models, 1)
	assert.Equal(t, "source/model", status.Models[0].Root)
	require.NotNil(t, status.Models[0].MaxModelLen)
	assert.EqualValues(t, 1048576, *status.Models[0].MaxModelLen)
	assert.Equal(t, inferenceEndpointStatus{Status: 403, Error: "http_error"}, status.Endpoints["/metrics"])
	assert.Empty(t, status.RawMetrics)
	assert.Empty(t, status.Metrics)
	body, err := common.Marshal(status)
	require.NoError(t, err)
	assert.NotContains(t, string(body), channel.Key)
}

func TestVLLMStatusRejectsInvalidTargetsAndRedirects(t *testing.T) {
	service.InitHttpClient()
	for _, base := range []string{"", "file:///etc/passwd", "http://user:secret@localhost", "http://localhost?key=secret"} {
		_, err := fetchInferenceStatus(context.Background(), &model.Channel{Type: constant.ChannelTypeVLLM, BaseURL: &base})
		require.Error(t, err)
		assert.NotContains(t, err.Error(), "secret")
	}
	_, err := fetchInferenceStatus(context.Background(), &model.Channel{Type: constant.ChannelTypeOllama})
	require.Error(t, err)
	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		t.Error("redirect must not forward channel credentials")
	}))
	defer target.Close()
	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		assert.Empty(t, r.Header.Get("Authorization"))
		http.Redirect(w, r, target.URL, http.StatusFound)
	}))
	defer upstream.Close()
	status, err := fetchInferenceStatus(context.Background(), &model.Channel{Type: constant.ChannelTypeVLLM, BaseURL: &upstream.URL, Key: "EMPTY"})
	require.NoError(t, err)
	for _, endpoint := range status.Endpoints {
		assert.Equal(t, inferenceEndpointStatus{Status: 302, Error: "http_error"}, endpoint)
	}
}

func TestVLLMStatusInvalidAndOversizedResponses(t *testing.T) {
	service.InitHttpClient()
	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch r.URL.Path {
		case "/health":
			w.WriteHeader(http.StatusServiceUnavailable)
		case "/version":
			_, _ = w.Write([]byte(`{"version":false}`))
		case "/v1/models":
			_, _ = w.Write([]byte(`{"data":[]}`))
		case "/metrics":
			_, _ = w.Write([]byte(strings.Repeat(" ", (4<<20)+1)))
		}
	}))
	defer upstream.Close()
	channel := &model.Channel{Type: constant.ChannelTypeVLLM, BaseURL: &upstream.URL, Key: "EMPTY"}
	status, err := fetchInferenceStatus(context.Background(), channel)
	require.NoError(t, err)
	assert.Equal(t, 503, status.Endpoints["/health"].Status)
	assert.Equal(t, "invalid_response", status.Endpoints["/version"].Error)
	assert.Empty(t, status.Endpoints["/v1/models"].Error)
	assert.Equal(t, "response_too_large", status.Endpoints["/metrics"].Error)
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	_, err = fetchInferenceStatus(ctx, channel)
	assert.ErrorIs(t, err, context.Canceled)
}

func TestParseVLLMMetricsPreservesSeriesAndHistogramTotals(t *testing.T) {
	metrics, err := parseInferenceMetrics(`# TYPE vllm:num_requests_running gauge
vllm:num_requests_running{engine="0",model_name="model,a"} 0
vllm:num_requests_running{model_name="model,a",engine="1"} 2
# TYPE vllm:time_to_first_token_seconds histogram
vllm:time_to_first_token_seconds_bucket{le="+Inf"} 4
vllm:time_to_first_token_seconds_sum 10
vllm:time_to_first_token_seconds_count 4
# TYPE vllm:kv_cache_usage_perc gauge
vllm:kv_cache_usage_perc NaN
vllm:bad_value +Inf
# TYPE process_start_time_seconds gauge
process_start_time_seconds 100
unrelated_metric 123
`, "vllm:")
	require.NoError(t, err)
	assert.ElementsMatch(t, []inferenceMetric{
		{Name: "vllm:num_requests_running", Labels: map[string]string{"engine": "0", "model_name": "model,a"}, Value: 0},
		{Name: "vllm:num_requests_running", Labels: map[string]string{"engine": "1", "model_name": "model,a"}, Value: 2},
		{Name: "vllm:time_to_first_token_seconds_sum", Labels: map[string]string{}, Value: 10},
		{Name: "vllm:time_to_first_token_seconds_count", Labels: map[string]string{}, Value: 4},
		{Name: "process_start_time_seconds", Labels: map[string]string{}, Value: 100},
	}, metrics)
	_, err = parseInferenceMetrics("vllm:broken{engine=oops} 1\n", "vllm:")
	require.Error(t, err)
}

// Opt-in verification against an operator-provided instance; credentials stay in the environment.
func TestVLLMStatusLive(t *testing.T) {
	base := os.Getenv("NEW_API_VLLM_TEST_URL")
	if base == "" {
		t.Skip("NEW_API_VLLM_TEST_URL is not configured")
	}
	service.InitHttpClient()
	status, err := fetchInferenceStatus(context.Background(), &model.Channel{
		Type: constant.ChannelTypeVLLM, BaseURL: &base, Key: os.Getenv("NEW_API_VLLM_TEST_KEY"),
	})
	require.NoError(t, err)
	for path, endpoint := range status.Endpoints {
		assert.Equal(t, 200, endpoint.Status, path)
		assert.Empty(t, endpoint.Error, path)
	}
	assert.NotEmpty(t, status.Version)
	assert.NotEmpty(t, status.Models)
	assert.NotEmpty(t, status.Metrics)
	assert.NotEmpty(t, status.RawMetrics)
}

func TestSGLangStatusUsesDocumentedEndpointsAndRedactsServerConfiguration(t *testing.T) {
	service.InitHttpClient()
	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		assert.Equal(t, http.MethodGet, r.Method)
		assert.Equal(t, "Bearer test-key", r.Header.Get("Authorization"))
		switch r.URL.Path {
		case "/health":
			w.WriteHeader(http.StatusOK)
		case "/server_info":
			_, _ = w.Write([]byte(`{"version":"0.5.19","api_key":"server-secret","admin_api_key":"admin-secret","internal_states":[{"private":"private-state"}]}`))
		case "/v1/models":
			_, _ = w.Write([]byte(`{"data":[{"id":"served-model","root":"served-model","max_model_len":32768}]}`))
		case "/metrics":
			_, _ = w.Write([]byte("# TYPE sglang:num_running_reqs gauge\nsglang:num_running_reqs{dp_rank=\"0\"} 0\nsglang:num_running_reqs{dp_rank=\"1\"} 2\n# TYPE sglang:time_to_first_token_seconds histogram\nsglang:time_to_first_token_seconds_bucket{le=\"+Inf\"} 2\nsglang:time_to_first_token_seconds_sum 1.5\nsglang:time_to_first_token_seconds_count 2\nvllm:num_requests_running 9\n"))
		default:
			t.Errorf("unexpected SGLang path: %s", r.URL.Path)
			w.WriteHeader(http.StatusNotFound)
		}
	}))
	defer upstream.Close()
	status, err := fetchInferenceStatus(context.Background(), &model.Channel{Type: constant.ChannelTypeSGLang, BaseURL: &upstream.URL, Key: "test-key"})
	require.NoError(t, err)
	assert.Equal(t, "0.5.19", status.Version)
	require.Len(t, status.Models, 1)
	assert.Equal(t, "served-model", status.Models[0].ID)
	assert.ElementsMatch(t, []inferenceMetric{
		{Name: "sglang:num_running_reqs", Labels: map[string]string{"dp_rank": "0"}, Value: 0},
		{Name: "sglang:num_running_reqs", Labels: map[string]string{"dp_rank": "1"}, Value: 2},
		{Name: "sglang:time_to_first_token_seconds_sum", Labels: map[string]string{}, Value: 1.5},
		{Name: "sglang:time_to_first_token_seconds_count", Labels: map[string]string{}, Value: 2},
	}, status.Metrics)
	encoded, err := common.Marshal(status)
	require.NoError(t, err)
	for _, secret := range []string{"test-key", "server-secret", "admin-secret", "private-state", "api_key", "internal_states"} {
		assert.NotContains(t, string(encoded), secret)
	}
}