wangbo
c89c56ca65
原因：线上 P24 同构验收中，单任务同步复制提交与全局容量锁串行化，使两个 Worker 的 48 个执行槽只能维持约 10–14 个运行任务，最老等待超过 15 分钟。 影响：新增可配置的 1–32 条准入微批次，默认 8；租约、任务准入与唯一 River job 在一个有界事务内原子提交，并按确定顺序预锁任务和容量范围，避免整窗 48 条大事务和多 Dispatcher 死锁。并容忍 rebind 已被其他 Dispatcher 完成的幂等竞态。 验证：Go 全量测试、go vet、真实 PostgreSQL 跨 Store 集成测试、ShellCheck、迁移安全检查、OpenAPI、前端 lint/test/build、Compose/Kubernetes 渲染及人工发布脚本均通过。
377 lines
15 KiB
Go
377 lines
15 KiB
Go
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestLoadIdentitySecretStoreUsesNewEnvironmentNamesAndIgnoresLegacyBusinessValues(t *testing.T) {
|
|
t.Setenv("IDENTITY_SECRET_STORE", "file")
|
|
t.Setenv("IDENTITY_SECRET_DIR", ".test-secrets/identity")
|
|
t.Setenv("OIDC_ISSUER", "https://legacy-sensitive.example/issuer")
|
|
t.Setenv("OIDC_INTROSPECTION_CLIENT_SECRET", "legacy-sensitive-secret")
|
|
t.Setenv("OIDC_SESSION_ENCRYPTION_KEY", "legacy-sensitive-session-key")
|
|
|
|
cfg := Load()
|
|
if cfg.IdentitySecretStore != "file" || cfg.IdentitySecretDir != ".test-secrets/identity" {
|
|
t.Fatalf("identity SecretStore = %q %q", cfg.IdentitySecretStore, cfg.IdentitySecretDir)
|
|
}
|
|
printed := fmt.Sprintf("%+v", cfg)
|
|
for _, legacyValue := range []string{"legacy-sensitive.example", "legacy-sensitive-secret", "legacy-sensitive-session-key"} {
|
|
if strings.Contains(printed, legacyValue) {
|
|
t.Fatalf("legacy identity business setting was loaded into Config: %q", legacyValue)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestValidateIdentityFileSecretStoreRequiresDirectory(t *testing.T) {
|
|
cfg := Config{
|
|
IdentitySecretStore: "file",
|
|
AsyncWorkerHardLimit: 2048,
|
|
AsyncWorkerInstanceHardLimit: 32,
|
|
AsyncWorkerRefreshIntervalSeconds: 5,
|
|
AsyncAdmissionMicrobatchSize: 8,
|
|
}
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "IDENTITY_SECRET_DIR") {
|
|
t.Fatalf("Validate() error = %v, want missing identity secret directory", err)
|
|
}
|
|
cfg.IdentitySecretDir = ".test-secrets/identity"
|
|
if err := cfg.Validate(); err != nil {
|
|
t.Fatalf("valid file SecretStore was rejected: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateIdentityKubernetesSecretStore(t *testing.T) {
|
|
cfg := Config{
|
|
IdentitySecretStore: "kubernetes",
|
|
IdentityKubernetesSecretName: "easyai-gateway-identity",
|
|
AsyncWorkerHardLimit: 2048,
|
|
AsyncWorkerInstanceHardLimit: 32,
|
|
AsyncWorkerRefreshIntervalSeconds: 5,
|
|
AsyncAdmissionMicrobatchSize: 8,
|
|
}
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "namespace") {
|
|
t.Fatalf("Validate() error = %v, want missing namespace", err)
|
|
}
|
|
cfg.IdentityKubernetesNamespace = "easyai"
|
|
if err := cfg.Validate(); err != nil {
|
|
t.Fatalf("valid Kubernetes SecretStore was rejected: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateIdentitySecurityEventTiming(t *testing.T) {
|
|
cfg := Config{
|
|
IdentitySecurityEventHeartbeatIntervalSeconds: 60,
|
|
IdentitySecurityEventStaleAfterSeconds: 60,
|
|
IdentitySecurityEventClockSkewSeconds: 60,
|
|
AsyncWorkerHardLimit: 2048,
|
|
AsyncWorkerInstanceHardLimit: 32,
|
|
AsyncWorkerRefreshIntervalSeconds: 5,
|
|
AsyncAdmissionMicrobatchSize: 8,
|
|
}
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "heartbeat") {
|
|
t.Fatalf("Validate() error = %v, want invalid stale threshold", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateAsyncWorkerSettings(t *testing.T) {
|
|
cfg := Config{
|
|
AsyncWorkerHardLimit: 10001,
|
|
AsyncWorkerInstanceHardLimit: 32,
|
|
AsyncWorkerRefreshIntervalSeconds: 5,
|
|
AsyncAdmissionMicrobatchSize: 8,
|
|
}
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "HARD_LIMIT") {
|
|
t.Fatalf("Validate() error = %v, want invalid hard limit", err)
|
|
}
|
|
cfg.AsyncWorkerHardLimit = 2048
|
|
cfg.AsyncWorkerInstanceHardLimit = 10001
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "INSTANCE_HARD_LIMIT") {
|
|
t.Fatalf("Validate() error = %v, want invalid instance hard limit", err)
|
|
}
|
|
cfg.AsyncWorkerInstanceHardLimit = 32
|
|
cfg.AsyncWorkerRefreshIntervalSeconds = 0
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "REFRESH_INTERVAL") {
|
|
t.Fatalf("Validate() error = %v, want invalid refresh interval", err)
|
|
}
|
|
t.Setenv("AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT", "not-an-integer")
|
|
loaded := Load()
|
|
if err := loaded.Validate(); err == nil || !strings.Contains(err.Error(), "HARD_LIMIT") {
|
|
t.Fatalf("Validate() error = %v, want invalid non-integer hard limit", err)
|
|
}
|
|
t.Setenv("AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT", "2048")
|
|
t.Setenv("AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT", "not-an-integer")
|
|
loaded = Load()
|
|
if err := loaded.Validate(); err == nil || !strings.Contains(err.Error(), "INSTANCE_HARD_LIMIT") {
|
|
t.Fatalf("Validate() error = %v, want invalid non-integer instance hard limit", err)
|
|
}
|
|
t.Setenv("AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT", "32")
|
|
t.Setenv("AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE", "33")
|
|
loaded = Load()
|
|
if err := loaded.Validate(); err == nil || !strings.Contains(err.Error(), "MICROBATCH_SIZE") {
|
|
t.Fatalf("Validate() error = %v, want invalid admission microbatch size", err)
|
|
}
|
|
}
|
|
|
|
func TestLoadAsyncAdmissionMicrobatchSize(t *testing.T) {
|
|
cfg := Load()
|
|
if cfg.AsyncAdmissionMicrobatchSize != 8 {
|
|
t.Fatalf("admission microbatch size=%d, want 8", cfg.AsyncAdmissionMicrobatchSize)
|
|
}
|
|
t.Setenv("AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE", "4")
|
|
cfg = Load()
|
|
if err := cfg.Validate(); err != nil {
|
|
t.Fatalf("valid admission microbatch size was rejected: %v", err)
|
|
}
|
|
if cfg.AsyncAdmissionMicrobatchSize != 4 {
|
|
t.Fatalf("admission microbatch size=%d, want 4", cfg.AsyncAdmissionMicrobatchSize)
|
|
}
|
|
}
|
|
|
|
func TestLoadAsyncQueueWorkerEnabled(t *testing.T) {
|
|
cfg := Load()
|
|
if !cfg.AsyncQueueWorkerEnabled {
|
|
t.Fatal("async queue worker must be enabled by default")
|
|
}
|
|
t.Setenv("AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED", "false")
|
|
cfg = Load()
|
|
if cfg.AsyncQueueWorkerEnabled {
|
|
t.Fatal("async queue worker remained enabled after explicit disable")
|
|
}
|
|
}
|
|
|
|
func TestProcessRolePrecedenceAndCompatibility(t *testing.T) {
|
|
t.Setenv("AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED", "false")
|
|
cfg := Load()
|
|
if got := cfg.EffectiveProcessRole(); got != "all" {
|
|
t.Fatalf("legacy configuration changed process role to %q", got)
|
|
}
|
|
if cfg.RunsAsyncExecutionWorker() {
|
|
t.Fatal("legacy async worker disable was ignored")
|
|
}
|
|
|
|
t.Setenv("AI_GATEWAY_PROCESS_ROLE", "worker")
|
|
cfg = Load()
|
|
if got := cfg.EffectiveProcessRole(); got != "worker" {
|
|
t.Fatalf("effective process role = %q, want worker", got)
|
|
}
|
|
if cfg.RunsPublicHTTP() || !cfg.RunsAsyncExecutionWorker() || !cfg.RunsBackgroundWorkers() {
|
|
t.Fatalf("worker role capabilities are inconsistent: %+v", cfg)
|
|
}
|
|
|
|
t.Setenv("AI_GATEWAY_PROCESS_ROLE", "api")
|
|
cfg = Load()
|
|
if !cfg.RunsPublicHTTP() || cfg.RunsAsyncExecutionWorker() || cfg.RunsBackgroundWorkers() {
|
|
t.Fatalf("api role capabilities are inconsistent: %+v", cfg)
|
|
}
|
|
|
|
t.Setenv("AI_GATEWAY_PROCESS_ROLE", "capacity-controller")
|
|
cfg = Load()
|
|
if cfg.RunsPublicHTTP() || cfg.RunsAsyncExecutionWorker() || cfg.RunsBackgroundWorkers() || !cfg.RunsCapacityController() {
|
|
t.Fatalf("capacity controller role capabilities are inconsistent: %+v", cfg)
|
|
}
|
|
}
|
|
|
|
func TestWorkerAutoscalingConfiguration(t *testing.T) {
|
|
t.Setenv("AI_GATEWAY_WORKER_AUTOSCALING_ENABLED", "true")
|
|
t.Setenv("AI_GATEWAY_WORKER_REPLICAS_NINGBO", "1")
|
|
t.Setenv("AI_GATEWAY_WORKER_REPLICAS_HONGKONG", "2")
|
|
t.Setenv("AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO", "1")
|
|
t.Setenv("AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG", "1")
|
|
t.Setenv("AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO", "2")
|
|
t.Setenv("AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG", "3")
|
|
cfg := Load()
|
|
if err := cfg.Validate(); err != nil {
|
|
t.Fatalf("valid autoscaling configuration was rejected: %v", err)
|
|
}
|
|
if !cfg.WorkerAutoscalingEnabled || cfg.WorkerReplicasHongkong != 2 || cfg.WorkerMaxReplicasHongkong != 3 {
|
|
t.Fatalf("autoscaling configuration=%+v", cfg)
|
|
}
|
|
cfg.WorkerReplicasHongkong = 4
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "bootstrap") {
|
|
t.Fatalf("Validate() error=%v, want bootstrap bound failure", err)
|
|
}
|
|
}
|
|
|
|
func TestLoadPrefersWorkerGlobalHardLimitAlias(t *testing.T) {
|
|
t.Setenv("AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT", "48")
|
|
t.Setenv("AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT", "96")
|
|
cfg := Load()
|
|
if cfg.AsyncWorkerHardLimit != 96 {
|
|
t.Fatalf("global worker hard limit=%d, want 96", cfg.AsyncWorkerHardLimit)
|
|
}
|
|
}
|
|
|
|
func TestValidatePostgresReplicaBudgets(t *testing.T) {
|
|
cfg := Load()
|
|
if cfg.PostgresConnectionBudget != 150 || cfg.PostgresNonWorkerConnectionBudget != 72 {
|
|
t.Fatalf(
|
|
"PostgreSQL budgets=%d/%d, want 150/72",
|
|
cfg.PostgresConnectionBudget,
|
|
cfg.PostgresNonWorkerConnectionBudget,
|
|
)
|
|
}
|
|
cfg.PostgresNonWorkerConnectionBudget = cfg.PostgresConnectionBudget
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "NON_WORKER") {
|
|
t.Fatalf("Validate() error=%v, want non-Worker budget failure", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateProcessRoleAndDatabasePool(t *testing.T) {
|
|
cfg := Load()
|
|
if cfg.DatabaseMinIdleConns != 0 ||
|
|
cfg.DatabaseMaxConnIdleSeconds != 0 ||
|
|
cfg.DatabaseIdleInTransactionTimeoutSeconds != 60 ||
|
|
cfg.DatabaseLockTimeoutSeconds != 30 {
|
|
t.Fatalf(
|
|
"database pool minimum/max-idle/transaction timeouts = %d/%d/%d/%d, want 0/0/60/30",
|
|
cfg.DatabaseMinIdleConns,
|
|
cfg.DatabaseMaxConnIdleSeconds,
|
|
cfg.DatabaseIdleInTransactionTimeoutSeconds,
|
|
cfg.DatabaseLockTimeoutSeconds,
|
|
)
|
|
}
|
|
cfg.ProcessRole = "invalid"
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "PROCESS_ROLE") {
|
|
t.Fatalf("Validate() error = %v, want invalid process role", err)
|
|
}
|
|
cfg.ProcessRole = "api"
|
|
cfg.DatabaseMaxConns = -1
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "DATABASE_MAX_CONNS") {
|
|
t.Fatalf("Validate() error = %v, want invalid database max conns", err)
|
|
}
|
|
cfg.DatabaseMaxConns = 16
|
|
cfg.DatabaseMinIdleConns = 17
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "DATABASE_MIN_IDLE_CONNS") {
|
|
t.Fatalf("Validate() error = %v, want invalid database min idle conns", err)
|
|
}
|
|
cfg.DatabaseMinIdleConns = 4
|
|
cfg.DatabaseCriticalMaxConns = 4
|
|
cfg.DatabaseRiverMaxConns = 12
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "River") {
|
|
t.Fatalf("Validate() error = %v, want database pool budget failure", err)
|
|
}
|
|
cfg.DatabaseRiverMaxConns = 4
|
|
t.Setenv("AI_GATEWAY_DATABASE_MIN_IDLE_CONNS", "not-an-integer")
|
|
if err := Load().Validate(); err == nil || !strings.Contains(err.Error(), "DATABASE_MIN_IDLE_CONNS") {
|
|
t.Fatalf("Validate() error = %v, want invalid non-integer database min idle conns", err)
|
|
}
|
|
t.Setenv("AI_GATEWAY_DATABASE_MIN_IDLE_CONNS", "4")
|
|
cfg.DatabaseMaxConnIdleSeconds = 3601
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "DATABASE_MAX_CONN_IDLE_SECONDS") {
|
|
t.Fatalf("Validate() error = %v, want invalid database max connection idle seconds", err)
|
|
}
|
|
cfg.DatabaseMaxConnIdleSeconds = 30
|
|
cfg.DatabaseIdleInTransactionTimeoutSeconds = 3601
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "IDLE_IN_TRANSACTION") {
|
|
t.Fatalf("Validate() error = %v, want invalid idle transaction timeout", err)
|
|
}
|
|
cfg.DatabaseIdleInTransactionTimeoutSeconds = 60
|
|
cfg.DatabaseLockTimeoutSeconds = 3601
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "LOCK_TIMEOUT") {
|
|
t.Fatalf("Validate() error = %v, want invalid lock timeout", err)
|
|
}
|
|
cfg.DatabaseLockTimeoutSeconds = 30
|
|
t.Setenv("AI_GATEWAY_DATABASE_LOCK_TIMEOUT_SECONDS", "not-an-integer")
|
|
if err := Load().Validate(); err == nil || !strings.Contains(err.Error(), "LOCK_TIMEOUT") {
|
|
t.Fatalf("Validate() error = %v, want invalid non-integer lock timeout", err)
|
|
}
|
|
t.Setenv("AI_GATEWAY_DATABASE_LOCK_TIMEOUT_SECONDS", "30")
|
|
cfg.MediaMaterializationConcurrency = 257
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "MEDIA_MATERIALIZATION_CONCURRENCY") {
|
|
t.Fatalf("Validate() error = %v, want invalid media materialization concurrency", err)
|
|
}
|
|
t.Setenv("AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY", "12")
|
|
if got := Load().MediaMaterializationConcurrency; got != 12 {
|
|
t.Fatalf("media materialization concurrency = %d, want 12", got)
|
|
}
|
|
cfg = Load()
|
|
cfg.MediaRequestConcurrency = 1025
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "MEDIA_REQUEST_CONCURRENCY") {
|
|
t.Fatalf("Validate() error = %v, want invalid media request concurrency", err)
|
|
}
|
|
t.Setenv("AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY", "96")
|
|
if got := Load().MediaRequestConcurrency; got != 96 {
|
|
t.Fatalf("media request concurrency = %d, want 96", got)
|
|
}
|
|
cfg = Load()
|
|
cfg.MediaImageNormalizationConcurrency = 65
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "MEDIA_IMAGE_NORMALIZATION_CONCURRENCY") {
|
|
t.Fatalf("Validate() error = %v, want invalid image normalization concurrency", err)
|
|
}
|
|
t.Setenv("AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY", "3")
|
|
if got := Load().MediaImageNormalizationConcurrency; got != 3 {
|
|
t.Fatalf("image normalization concurrency = %d, want 3", got)
|
|
}
|
|
}
|
|
|
|
func TestValidateTaskHistorySettings(t *testing.T) {
|
|
cfg := Load()
|
|
cfg.TaskRetentionDays = 30
|
|
cfg.TaskAnalysisRetentionDays = 31
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "ANALYSIS_RETENTION") {
|
|
t.Fatalf("Validate() error = %v, want invalid analysis retention", err)
|
|
}
|
|
cfg.TaskAnalysisRetentionDays = 7
|
|
cfg.TaskCleanupIntervalSeconds = 59
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "CLEANUP_INTERVAL") {
|
|
t.Fatalf("Validate() error = %v, want invalid cleanup interval", err)
|
|
}
|
|
cfg.TaskCleanupIntervalSeconds = 300
|
|
cfg.TaskCleanupBatchSize = 5001
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "CLEANUP_BATCH") {
|
|
t.Fatalf("Validate() error = %v, want invalid cleanup batch", err)
|
|
}
|
|
}
|
|
|
|
func TestLoadAndValidateDirectMediaOSS(t *testing.T) {
|
|
t.Setenv("APP_ENV", "production")
|
|
t.Setenv("AI_GATEWAY_MEDIA_OSS_DIRECT_ENABLED", "true")
|
|
t.Setenv("AI_GATEWAY_MEDIA_OSS_ENDPOINT", "https://media-bucket.oss-cn-shanghai.aliyuncs.com")
|
|
t.Setenv("AI_GATEWAY_MEDIA_OSS_BUCKET", "media-bucket")
|
|
t.Setenv("AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_ID", "test-access-key")
|
|
t.Setenv("AI_GATEWAY_MEDIA_OSS_ACCESS_KEY_SECRET", "test-access-secret")
|
|
t.Setenv("AI_GATEWAY_MEDIA_OSS_PUBLIC_BASE_URL", "https://cdn.example.com")
|
|
t.Setenv("AI_GATEWAY_MEDIA_OSS_OBJECT_PREFIX", "easyai-ai-gateway/production/media")
|
|
|
|
cfg := Load()
|
|
if !cfg.MediaOSSDirectEnabled || cfg.MediaOSSBucket != "media-bucket" ||
|
|
cfg.MediaOSSObjectPrefix != "easyai-ai-gateway/production/media" {
|
|
t.Fatalf("unexpected direct media OSS config: %+v", cfg)
|
|
}
|
|
if err := cfg.Validate(); err != nil {
|
|
t.Fatalf("validate direct media OSS: %v", err)
|
|
}
|
|
cfg.MediaOSSAccessKeySecret = ""
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "ACCESS_KEY_SECRET") {
|
|
t.Fatalf("Validate() error = %v, want missing direct OSS secret", err)
|
|
}
|
|
cfg.MediaOSSAccessKeySecret = "test-access-secret"
|
|
cfg.MediaOSSEndpoint = "http://media-bucket.example.com"
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "HTTPS URL") {
|
|
t.Fatalf("Validate() error = %v, want insecure production endpoint", err)
|
|
}
|
|
cfg.MediaOSSEndpoint = "https://media-bucket.example.com"
|
|
cfg.MediaOSSObjectPrefix = "../outside"
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "OBJECT_PREFIX") {
|
|
t.Fatalf("Validate() error = %v, want invalid object prefix", err)
|
|
}
|
|
}
|
|
|
|
func TestCapacityControllerDoesNotRequireMediaOSSCredentials(t *testing.T) {
|
|
cfg := Load()
|
|
cfg.ProcessRole = "capacity-controller"
|
|
cfg.MediaOSSDirectEnabled = true
|
|
cfg.MediaOSSAccessKeyID = ""
|
|
cfg.MediaOSSAccessKeySecret = ""
|
|
if err := cfg.Validate(); err != nil {
|
|
t.Fatalf("capacity controller rejected unused media OSS credentials: %v", err)
|
|
}
|
|
|
|
cfg.ProcessRole = "worker"
|
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "ACCESS_KEY") {
|
|
t.Fatalf("worker Validate() error = %v, want missing direct OSS credentials", err)
|
|
}
|
|
}
|