chore(git): 合并远端主分支最新变更
This commit is contained in:
@@ -54,6 +54,7 @@ func TestAsyncWorkerDynamicConcurrencyAcceptance(t *testing.T) {
|
||||
JWTSecret: "test-secret",
|
||||
BillingEngineMode: "observe",
|
||||
CORSAllowedOrigin: "*",
|
||||
AsyncQueueWorkerEnabled: true,
|
||||
AsyncWorkerHardLimit: 256,
|
||||
AsyncWorkerRefreshIntervalSeconds: 1,
|
||||
}, db, slog.New(slog.NewTextHandler(io.Discard, nil))))
|
||||
|
||||
@@ -122,8 +122,11 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
|
||||
server.auth.LocalAPIKeyVerifier = db.VerifyLocalAPIKey
|
||||
if cfg.AsyncQueueWorkerEnabled {
|
||||
server.runner.StartAsyncQueueWorker(ctx)
|
||||
} else if logger != nil {
|
||||
logger.Info("asynchronous queue worker disabled for this process")
|
||||
} else {
|
||||
server.runner.StartAsyncQueueClient(ctx)
|
||||
if logger != nil {
|
||||
logger.Info("asynchronous queue worker disabled for this process")
|
||||
}
|
||||
}
|
||||
server.runner.StartBillingSettlementWorker(ctx)
|
||||
server.runner.StartTaskHistoryWorkers(ctx)
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
func TestAsyncQueueClientEnqueuesWithoutExecutionWorker(t *testing.T) {
|
||||
databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run the queue client integration test")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
db, err := store.Connect(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("connect store: %v", err)
|
||||
}
|
||||
t.Cleanup(db.Close)
|
||||
|
||||
suffix := strconv.FormatInt(time.Now().UnixNano(), 10)
|
||||
task, err := db.CreateTask(ctx, store.CreateTaskInput{
|
||||
Kind: "images.edits",
|
||||
Model: "queue-client-" + suffix,
|
||||
Request: map[string]any{"prompt": "enqueue without execution worker"},
|
||||
Async: true,
|
||||
RunMode: "simulation",
|
||||
}, &auth.User{ID: "queue-client-" + suffix, Source: "gateway"})
|
||||
if err != nil {
|
||||
t.Fatalf("create task: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = db.Pool().Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id = $1::uuid`, task.ID)
|
||||
})
|
||||
|
||||
service := New(config.Config{AppEnv: "test"}, db, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
service.StartAsyncQueueClient(ctx)
|
||||
if err := service.EnqueueAsyncTask(ctx, task); err != nil {
|
||||
t.Fatalf("enqueue task without execution worker: %v", err)
|
||||
}
|
||||
|
||||
queued, err := db.GetTask(ctx, task.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("load queued task: %v", err)
|
||||
}
|
||||
if queued.RiverJobID <= 0 {
|
||||
t.Fatal("queue client did not persist a River job ID")
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = db.Pool().Exec(context.Background(), `DELETE FROM river_job WHERE id = $1`, queued.RiverJobID)
|
||||
})
|
||||
|
||||
var attemptedByCount int
|
||||
if err := db.Pool().QueryRow(ctx, `
|
||||
SELECT cardinality(attempted_by)
|
||||
FROM river_job
|
||||
WHERE id = $1`, queued.RiverJobID).Scan(&attemptedByCount); err != nil {
|
||||
t.Fatalf("load River job: %v", err)
|
||||
}
|
||||
if attemptedByCount != 0 {
|
||||
t.Fatalf("control-only queue client executed the job: attempted_by=%d", attemptedByCount)
|
||||
}
|
||||
}
|
||||
@@ -78,13 +78,20 @@ func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs
|
||||
}
|
||||
|
||||
func (s *Service) StartAsyncQueueWorker(ctx context.Context) {
|
||||
if err := s.startRiverQueue(ctx); err != nil {
|
||||
if err := s.startRiverQueue(ctx, true); err != nil {
|
||||
s.logger.Error("start river async queue failed", "error", err)
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) startRiverQueue(ctx context.Context) error {
|
||||
func (s *Service) StartAsyncQueueClient(ctx context.Context) {
|
||||
if err := s.startRiverQueue(ctx, false); err != nil {
|
||||
s.logger.Error("start river async queue client failed", "error", err)
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) startRiverQueue(ctx context.Context, workerEnabled bool) error {
|
||||
driver := riverpgxv5.New(s.store.Pool())
|
||||
migrator, err := rivermigrate.New(driver, nil)
|
||||
if err != nil {
|
||||
@@ -102,6 +109,23 @@ func (s *Service) startRiverQueue(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !workerEnabled {
|
||||
s.riverMu.Lock()
|
||||
s.riverControlClient = controlClient
|
||||
s.riverExecutionClient = nil
|
||||
s.riverWorkerCapacity = 0
|
||||
s.riverDrainingClients = make(map[asyncExecutionClient]struct{})
|
||||
s.riverMu.Unlock()
|
||||
if err := s.recoverAsyncRiverJobs(ctx); err != nil {
|
||||
s.riverMu.Lock()
|
||||
s.riverControlClient = nil
|
||||
s.riverMu.Unlock()
|
||||
return err
|
||||
}
|
||||
go s.stopAsyncWorkersOnShutdown(ctx)
|
||||
s.logger.Info("river async queue client initialized without execution workers")
|
||||
return nil
|
||||
}
|
||||
snapshot, err := s.loadAsyncWorkerCapacity(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("calculate initial async worker capacity: %w", err)
|
||||
@@ -271,6 +295,7 @@ func (s *Service) drainAsyncWorkerClient(client asyncExecutionClient) {
|
||||
func (s *Service) stopAsyncWorkersOnShutdown(ctx context.Context) {
|
||||
<-ctx.Done()
|
||||
s.riverMu.Lock()
|
||||
s.riverControlClient = nil
|
||||
clients := make([]asyncExecutionClient, 0, 1+len(s.riverDrainingClients))
|
||||
if s.riverExecutionClient != nil {
|
||||
clients = append(clients, s.riverExecutionClient)
|
||||
|
||||
@@ -64,14 +64,24 @@ wait_for_url() {
|
||||
verify_site() {
|
||||
local site=$1
|
||||
local address
|
||||
local api_port
|
||||
local web_port
|
||||
case $site in
|
||||
hongkong) address=10.77.0.2 ;;
|
||||
ningbo) address=10.77.0.1 ;;
|
||||
hongkong)
|
||||
address=10.77.0.2
|
||||
api_port=31089
|
||||
web_port=31179
|
||||
;;
|
||||
ningbo)
|
||||
address=10.77.0.1
|
||||
api_port=31088
|
||||
web_port=31178
|
||||
;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
wait_for_url "$site API health" "http://$address:30088/api/v1/healthz" easyai-ai-gateway
|
||||
wait_for_url "$site API readiness" "http://$address:30088/api/v1/readyz" '"ok":true'
|
||||
wait_for_url "$site Web" "http://$address:30178/" 'EasyAI AI Gateway'
|
||||
wait_for_url "$site API health" "http://$address:$api_port/api/v1/healthz" easyai-ai-gateway
|
||||
wait_for_url "$site API readiness" "http://$address:$api_port/api/v1/readyz" '"ok":true'
|
||||
wait_for_url "$site Web" "http://$address:$web_port/" 'EasyAI AI Gateway'
|
||||
}
|
||||
|
||||
rollout_site() {
|
||||
|
||||
@@ -32,6 +32,10 @@ spec:
|
||||
easyai.io/workload: "true"
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
@@ -134,6 +138,10 @@ spec:
|
||||
easyai.io/workload: "true"
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
@@ -418,6 +426,90 @@ spec:
|
||||
targetPort: http
|
||||
nodePort: 30178
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: api-ningbo-edge
|
||||
labels:
|
||||
app.kubernetes.io/name: easyai-api
|
||||
app.kubernetes.io/component: api
|
||||
app.kubernetes.io/part-of: easyai-ai-gateway
|
||||
easyai.io/site: ningbo
|
||||
spec:
|
||||
type: NodePort
|
||||
externalTrafficPolicy: Local
|
||||
selector:
|
||||
app.kubernetes.io/name: easyai-api
|
||||
easyai.io/site: ningbo
|
||||
ports:
|
||||
- name: http
|
||||
port: 8088
|
||||
targetPort: http
|
||||
nodePort: 31088
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: api-hongkong-edge
|
||||
labels:
|
||||
app.kubernetes.io/name: easyai-api
|
||||
app.kubernetes.io/component: api
|
||||
app.kubernetes.io/part-of: easyai-ai-gateway
|
||||
easyai.io/site: hongkong
|
||||
spec:
|
||||
type: NodePort
|
||||
externalTrafficPolicy: Local
|
||||
selector:
|
||||
app.kubernetes.io/name: easyai-api
|
||||
easyai.io/site: hongkong
|
||||
ports:
|
||||
- name: http
|
||||
port: 8088
|
||||
targetPort: http
|
||||
nodePort: 31089
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: web-ningbo-edge
|
||||
labels:
|
||||
app.kubernetes.io/name: easyai-web
|
||||
app.kubernetes.io/component: web
|
||||
app.kubernetes.io/part-of: easyai-ai-gateway
|
||||
easyai.io/site: ningbo
|
||||
spec:
|
||||
type: NodePort
|
||||
externalTrafficPolicy: Local
|
||||
selector:
|
||||
app.kubernetes.io/name: easyai-web
|
||||
easyai.io/site: ningbo
|
||||
ports:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: http
|
||||
nodePort: 31178
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: web-hongkong-edge
|
||||
labels:
|
||||
app.kubernetes.io/name: easyai-web
|
||||
app.kubernetes.io/component: web
|
||||
app.kubernetes.io/part-of: easyai-ai-gateway
|
||||
easyai.io/site: hongkong
|
||||
spec:
|
||||
type: NodePort
|
||||
externalTrafficPolicy: Local
|
||||
selector:
|
||||
app.kubernetes.io/name: easyai-web
|
||||
easyai.io/site: hongkong
|
||||
ports:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: http
|
||||
nodePort: 31179
|
||||
---
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
upstream easyai_gateway_api {
|
||||
least_conn;
|
||||
keepalive 64;
|
||||
server 10.77.0.1:30088 max_fails=2 fail_timeout=5s;
|
||||
server 10.77.0.2:30088 max_fails=2 fail_timeout=5s;
|
||||
server 10.77.0.1:31088 max_fails=2 fail_timeout=5s;
|
||||
server 10.77.0.2:31089 max_fails=2 fail_timeout=5s;
|
||||
}
|
||||
|
||||
upstream easyai_gateway_web {
|
||||
least_conn;
|
||||
keepalive 32;
|
||||
server 10.77.0.1:30178 max_fails=2 fail_timeout=5s;
|
||||
server 10.77.0.2:30178 max_fails=2 fail_timeout=5s;
|
||||
server 10.77.0.1:31178 max_fails=2 fail_timeout=5s;
|
||||
server 10.77.0.2:31179 max_fails=2 fail_timeout=5s;
|
||||
}
|
||||
|
||||
server {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Private-only deterministic path used by the cross-node acceptance test.
|
||||
server {
|
||||
listen 10.77.0.1:18088;
|
||||
listen 10.77.0.1:18089;
|
||||
server_name _;
|
||||
|
||||
allow 10.77.0.0/24;
|
||||
@@ -19,6 +19,6 @@ server {
|
||||
proxy_read_timeout 3600s;
|
||||
proxy_request_buffering off;
|
||||
proxy_buffering off;
|
||||
proxy_pass http://10.77.0.1:30088;
|
||||
proxy_pass http://10.77.0.1:31088;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,15 +57,17 @@ operator_checksums=(
|
||||
e0c5ff41bf5b01c0775bf225929b8423d543cd23e9742e4274fe7de8499cf93a
|
||||
d2e71e7b06822448f1a421f05781846cfdb9cc621e7ef32eef5e20c5133213b0
|
||||
)
|
||||
for index in 0 1 2; do
|
||||
operator=${operators[$index]}
|
||||
curl -fsSL "${operator_urls[$index]}" -o "$working_directory/$operator"
|
||||
actual_checksum=$(shasum -a 256 "$working_directory/$operator" | awk '{print $1}')
|
||||
[[ $actual_checksum == "${operator_checksums[$index]}" ]] || {
|
||||
echo "operator manifest checksum mismatch: $operator" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
if [[ $mode == prepare ]]; then
|
||||
for index in 0 1 2; do
|
||||
operator=${operators[$index]}
|
||||
curl -fsSL "${operator_urls[$index]}" -o "$working_directory/$operator"
|
||||
actual_checksum=$(shasum -a 256 "$working_directory/$operator" | awk '{print $1}')
|
||||
[[ $actual_checksum == "${operator_checksums[$index]}" ]] || {
|
||||
echo "operator manifest checksum mismatch: $operator" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
fi
|
||||
|
||||
cp -R "$cluster_root/deploy/kubernetes/production" "$working_directory/production"
|
||||
api_digest=${api_image##*@}
|
||||
@@ -126,7 +128,11 @@ stringData:
|
||||
EOF
|
||||
chmod 0600 "$working_directory/bootstrap-secrets.yaml"
|
||||
|
||||
for file in cert-manager.yaml cnpg.yaml barman.yaml easyai-production.yaml bootstrap-secrets.yaml; do
|
||||
platform_files=(easyai-production.yaml bootstrap-secrets.yaml)
|
||||
if [[ $mode == prepare ]]; then
|
||||
platform_files=(cert-manager.yaml cnpg.yaml barman.yaml "${platform_files[@]}")
|
||||
fi
|
||||
for file in "${platform_files[@]}"; do
|
||||
cluster_scp "$working_directory/$file" "$CLUSTER_NINGBO_HOST:/root/$file"
|
||||
done
|
||||
|
||||
@@ -151,14 +157,16 @@ cleanup_bootstrap_files() {
|
||||
}
|
||||
trap cleanup_bootstrap_files EXIT HUP INT TERM
|
||||
|
||||
$kubectl apply --server-side --force-conflicts -f /root/cert-manager.yaml >/dev/null
|
||||
$kubectl rollout status deployment/cert-manager -n cert-manager --timeout=180s
|
||||
$kubectl rollout status deployment/cert-manager-webhook -n cert-manager --timeout=180s
|
||||
$kubectl rollout status deployment/cert-manager-cainjector -n cert-manager --timeout=180s
|
||||
$kubectl apply --server-side --force-conflicts -f /root/cnpg.yaml >/dev/null
|
||||
$kubectl rollout status deployment/cnpg-controller-manager -n cnpg-system --timeout=600s
|
||||
$kubectl apply --server-side --force-conflicts -f /root/barman.yaml >/dev/null
|
||||
$kubectl rollout status deployment/barman-cloud -n cnpg-system --timeout=600s
|
||||
if [[ $mode == prepare ]]; then
|
||||
$kubectl apply --server-side --force-conflicts -f /root/cert-manager.yaml >/dev/null
|
||||
$kubectl rollout status deployment/cert-manager -n cert-manager --timeout=180s
|
||||
$kubectl rollout status deployment/cert-manager-webhook -n cert-manager --timeout=180s
|
||||
$kubectl rollout status deployment/cert-manager-cainjector -n cert-manager --timeout=180s
|
||||
$kubectl apply --server-side --force-conflicts -f /root/cnpg.yaml >/dev/null
|
||||
$kubectl rollout status deployment/cnpg-controller-manager -n cnpg-system --timeout=600s
|
||||
$kubectl apply --server-side --force-conflicts -f /root/barman.yaml >/dev/null
|
||||
$kubectl rollout status deployment/barman-cloud -n cnpg-system --timeout=600s
|
||||
fi
|
||||
|
||||
$kubectl create namespace easyai --dry-run=client -o yaml | $kubectl apply -f - >/dev/null
|
||||
$kubectl apply -f /root/bootstrap-secrets.yaml >/dev/null
|
||||
|
||||
@@ -51,11 +51,26 @@ function createFixturePNG(width = 256, height = 256) {
|
||||
])
|
||||
}
|
||||
|
||||
const baseURL = process.env.AI_GATEWAY_E2E_BASE_URL || 'http://10.77.0.1:18088'
|
||||
const baseURL = process.env.AI_GATEWAY_E2E_BASE_URL || 'http://10.77.0.1:18089'
|
||||
const account = Buffer.from(process.env.E2E_ACCOUNT_B64 || '', 'base64').toString('utf8')
|
||||
const password = Buffer.from(process.env.E2E_PASSWORD_B64 || '', 'base64').toString('utf8')
|
||||
const model = process.env.AI_GATEWAY_E2E_IMAGE_MODEL || 'openai:gpt-image-1'
|
||||
const imageSize = process.env.AI_GATEWAY_E2E_IMAGE_SIZE || '2048x2048'
|
||||
assert(account && password, 'E2E login credentials are unavailable')
|
||||
assert(/^\d+x\d+$/.test(imageSize), 'E2E image size is invalid')
|
||||
const [imageWidth, imageHeight] = imageSize.split('x').map(Number)
|
||||
const imageResolution =
|
||||
process.env.AI_GATEWAY_E2E_IMAGE_RESOLUTION ||
|
||||
(imageWidth === imageHeight && imageWidth % 1024 === 0 ? `${imageWidth / 1024}K` : '')
|
||||
const imageAspectRatio =
|
||||
process.env.AI_GATEWAY_E2E_IMAGE_ASPECT_RATIO ||
|
||||
`${imageWidth / greatestCommonDivisor(imageWidth, imageHeight)}:${imageHeight / greatestCommonDivisor(imageWidth, imageHeight)}`
|
||||
assert(imageResolution, 'E2E image resolution is unavailable')
|
||||
|
||||
function greatestCommonDivisor(left, right) {
|
||||
while (right !== 0) [left, right] = [right, left % right]
|
||||
return left
|
||||
}
|
||||
|
||||
async function request(path, { method = 'GET', token, body, form } = {}) {
|
||||
const headers = { Host: 'ai.51easyai.com' }
|
||||
@@ -95,6 +110,13 @@ const login = await request('/api/v1/auth/login', {
|
||||
const jwt = login.payload.accessToken
|
||||
assert(typeof jwt === 'string' && jwt.length > 20, 'login did not return an access token')
|
||||
|
||||
const existingKeys = await request('/api/v1/api-keys', { token: jwt })
|
||||
for (const item of existingKeys.payload.items || []) {
|
||||
if (typeof item?.id === 'string' && item?.name?.startsWith('cluster-cross-node-')) {
|
||||
await request(`/api/v1/api-keys/${item.id}`, { method: 'DELETE', token: jwt })
|
||||
}
|
||||
}
|
||||
|
||||
const keyName = `cluster-cross-node-${Date.now()}`
|
||||
const keyResult = await request('/api/v1/api-keys', {
|
||||
method: 'POST',
|
||||
@@ -102,18 +124,61 @@ const keyResult = await request('/api/v1/api-keys', {
|
||||
body: { name: keyName },
|
||||
})
|
||||
const apiKey = keyResult.payload.secret
|
||||
const apiKeyId = keyResult.payload.id || keyResult.payload.item?.id
|
||||
const apiKeyId = keyResult.payload.apiKey?.id
|
||||
assert(typeof apiKey === 'string' && apiKey.startsWith('sk-gw-'), 'API key creation failed')
|
||||
assert(typeof apiKeyId === 'string' && apiKeyId.length > 0, 'API key ID is unavailable')
|
||||
|
||||
let result
|
||||
try {
|
||||
const assignableModels = await request('/api/v1/api-keys/assignable-models', { token: jwt })
|
||||
const targetModels = (assignableModels.payload.items || []).filter(
|
||||
(item) =>
|
||||
[item?.modelName, item?.modelAlias].includes(model) &&
|
||||
Array.isArray(item?.modelType) &&
|
||||
item.modelType.includes('image_edit'),
|
||||
)
|
||||
assert(targetModels.length > 0, `no assignable image_edit platform model found for ${model}`)
|
||||
const resourceMap = new Map()
|
||||
for (const item of targetModels) {
|
||||
for (const [resourceType, resourceId] of [
|
||||
['platform', item.platformId],
|
||||
['platform_model', item.id],
|
||||
['base_model', item.baseModelId],
|
||||
]) {
|
||||
if (typeof resourceId === 'string' && resourceId) {
|
||||
resourceMap.set(`${resourceType}:${resourceId}`, {
|
||||
resourceType,
|
||||
resourceId,
|
||||
priority: 100,
|
||||
minPermissionLevel: 0,
|
||||
status: 'active',
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
await request('/api/v1/api-keys/access-rules/batch', {
|
||||
method: 'POST',
|
||||
token: jwt,
|
||||
body: {
|
||||
subjectType: 'api_key',
|
||||
subjectId: apiKeyId,
|
||||
effect: 'allow',
|
||||
upsertResources: [...resourceMap.values()],
|
||||
deleteResources: [],
|
||||
},
|
||||
})
|
||||
|
||||
const fixture = createFixturePNG()
|
||||
const sourceSHA256 = createHash('sha256').update(fixture).digest('hex')
|
||||
const form = new FormData()
|
||||
form.append('model', model)
|
||||
form.append('prompt', `Cross-node HA acceptance ${randomUUID()}: add a thin white border`)
|
||||
form.append('size', '1024x1024')
|
||||
form.append('image', new Blob([fixture], { type: 'image/png' }), 'cluster-e2e.png')
|
||||
form.append('size', imageSize)
|
||||
form.append('width', String(imageWidth))
|
||||
form.append('height', String(imageHeight))
|
||||
form.append('resolution', imageResolution)
|
||||
form.append('aspect_ratio', imageAspectRatio)
|
||||
form.append('images', new Blob([fixture], { type: 'image/png' }), 'cluster-e2e.png')
|
||||
const accepted = await request('/api/v1/images/edits', {
|
||||
method: 'POST',
|
||||
token: apiKey,
|
||||
|
||||
@@ -51,10 +51,16 @@ case $drill in
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$new_primary" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
primary=$1
|
||||
cleanup_probe() {
|
||||
k3s kubectl exec -n easyai "$primary" -c postgres -- \
|
||||
psql -X -U postgres -d easyai_ai_gateway -c \
|
||||
"DELETE FROM system_settings WHERE setting_key='ha_acceptance_probe';" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup_probe EXIT HUP INT TERM
|
||||
k3s kubectl exec -n easyai "$primary" -c postgres -- \
|
||||
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -c \
|
||||
"INSERT INTO system_settings(setting_key,value) VALUES('ha_acceptance_probe',jsonb_build_object('at',now())) ON CONFLICT(setting_key) DO UPDATE SET value=excluded.value,updated_at=now();" >/dev/null
|
||||
for _ in $(seq 1 180); do
|
||||
for _ in $(seq 1 900); do
|
||||
if [[ $(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') == 2 ]]; then
|
||||
replication=$(k3s kubectl exec -n easyai "$primary" -c postgres -- \
|
||||
psql -X -U postgres -d easyai_ai_gateway -At -c \
|
||||
@@ -67,6 +73,7 @@ done
|
||||
k3s kubectl exec -n easyai "$primary" -c postgres -- \
|
||||
psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -c \
|
||||
"DELETE FROM system_settings WHERE setting_key='ha_acceptance_probe';" >/dev/null
|
||||
trap - EXIT HUP INT TERM
|
||||
REMOTE
|
||||
echo "database_failure_drill=PASS promotion_seconds=$elapsed old_primary=$old_primary new_primary=$new_primary"
|
||||
;;
|
||||
|
||||
@@ -81,11 +81,10 @@ if [[ $mode == activate ]]; then
|
||||
for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"; do
|
||||
cluster_ssh "$host" 'bash -s' <<'REMOTE'
|
||||
set -euo pipefail
|
||||
ln -sfn /etc/nginx/sites-available/ai.51easyai.com-cluster \
|
||||
install -m 0644 /etc/nginx/sites-available/ai.51easyai.com-cluster \
|
||||
/etc/nginx/conf.d/easyai-ai-gateway-ai.51easyai.com.conf
|
||||
rm -f /etc/nginx/sites-enabled/ai.51easyai.com \
|
||||
/etc/nginx/sites-enabled/ai.51easyai.com-cluster
|
||||
if [[ -L /etc/nginx/sites-enabled/ai.51easyai.com ]]; then
|
||||
rm -f /etc/nginx/sites-enabled/ai.51easyai.com
|
||||
fi
|
||||
nginx -t
|
||||
systemctl reload nginx
|
||||
REMOTE
|
||||
|
||||
@@ -101,7 +101,7 @@ set -euo pipefail
|
||||
|
||||
source_sha=$1
|
||||
api_image=$2
|
||||
active_site=${AI_GATEWAY_NGINX_ACTIVE_SITE:-/etc/nginx/sites-enabled/ai.51easyai.com}
|
||||
active_site=${AI_GATEWAY_NGINX_ACTIVE_SITE:-/etc/nginx/conf.d/easyai-ai-gateway-ai.51easyai.com.conf}
|
||||
timestamp=$(date -u '+%Y%m%dT%H%M%SZ')
|
||||
backup_directory=/var/backups/easyai-ai-gateway/pre-cutover-"$timestamp"
|
||||
old_site_backup=$backup_directory/nginx-site.conf
|
||||
@@ -118,9 +118,7 @@ rollback_before_commit() {
|
||||
--replicas=0 >/dev/null 2>&1 || true
|
||||
docker start easyai-ai-gateway-api-1 easyai-ai-gateway-web-1 >/dev/null 2>&1 || true
|
||||
if [[ -f $old_site_backup ]]; then
|
||||
install -m 0644 "$old_site_backup" /etc/nginx/sites-available/ai.51easyai.com-pre-ha
|
||||
rm -f -- "$active_site"
|
||||
ln -s /etc/nginx/sites-available/ai.51easyai.com-pre-ha "$active_site"
|
||||
install -m 0644 "$old_site_backup" "$active_site"
|
||||
nginx -t && systemctl reload nginx
|
||||
fi
|
||||
else
|
||||
@@ -137,13 +135,20 @@ install -d -m 0700 "$backup_directory"
|
||||
}
|
||||
cp -L "$active_site" "$old_site_backup"
|
||||
chmod 0600 "$old_site_backup"
|
||||
rm -f -- "$active_site"
|
||||
ln -s /etc/nginx/sites-available/ai.51easyai.com-maintenance "$active_site"
|
||||
install -m 0644 /etc/nginx/sites-available/ai.51easyai.com-maintenance "$active_site"
|
||||
nginx -t
|
||||
systemctl reload nginx
|
||||
curl -ksS --resolve ai.51easyai.com:443:127.0.0.1 \
|
||||
-o /dev/null -w '%{http_code}' https://ai.51easyai.com/api/v1/healthz |
|
||||
grep -qx 503
|
||||
maintenance_code=
|
||||
for _ in $(seq 1 15); do
|
||||
maintenance_code=$(curl -ksS --resolve ai.51easyai.com:443:127.0.0.1 \
|
||||
-o /dev/null -w '%{http_code}' https://ai.51easyai.com/api/v1/healthz || true)
|
||||
[[ $maintenance_code == 503 ]] && break
|
||||
sleep 1
|
||||
done
|
||||
[[ $maintenance_code == 503 ]] || {
|
||||
echo "maintenance NGINX did not become active: HTTP $maintenance_code" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
docker stop --time 60 easyai-ai-gateway-api-1 >/dev/null
|
||||
postgres_user=$(docker inspect easyai-ai-gateway-postgres-1 \
|
||||
@@ -159,7 +164,7 @@ snapshot_sql=$backup_directory/acceptance-snapshot.sql
|
||||
cat >"$snapshot_sql" <<'SQL'
|
||||
\pset tuples_only on
|
||||
\pset format unaligned
|
||||
SELECT 'server_version=' || current_setting('server_version');
|
||||
SELECT 'server_version_num=' || current_setting('server_version_num');
|
||||
SELECT 'extensions=' || COALESCE(string_agg(extname || ':' || extversion, ',' ORDER BY extname), '')
|
||||
FROM pg_extension WHERE extname <> 'plpgsql';
|
||||
SELECT 'tables=' || count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
@@ -187,7 +192,7 @@ SELECT 'gateway_wallet_transactions=' || count(*) FROM gateway_wallet_transactio
|
||||
SELECT 'gateway_upload_assets=' || count(*) FROM gateway_upload_assets;
|
||||
SELECT 'gateway_users=' || count(*) FROM gateway_users;
|
||||
SELECT 'gateway_api_keys=' || count(*) FROM gateway_api_keys;
|
||||
SELECT 'gateway_providers=' || count(*) FROM gateway_providers;
|
||||
SELECT 'model_catalog_providers=' || count(*) FROM model_catalog_providers;
|
||||
SELECT 'task_sample=' || md5(COALESCE(string_agg(id::text, ',' ORDER BY id), ''))
|
||||
FROM (SELECT id FROM gateway_tasks ORDER BY id LIMIT 100) sample;
|
||||
SQL
|
||||
@@ -310,11 +315,8 @@ k3s kubectl scale deployment -n easyai \
|
||||
easyai-api-ningbo easyai-api-hongkong easyai-web-ningbo easyai-web-hongkong \
|
||||
--replicas=0 >/dev/null 2>&1 || true
|
||||
docker start easyai-ai-gateway-api-1 easyai-ai-gateway-web-1 >/dev/null
|
||||
rm -f -- /etc/nginx/sites-enabled/ai.51easyai.com
|
||||
install -m 0644 "$backup_directory/nginx-site.conf" \
|
||||
/etc/nginx/sites-available/ai.51easyai.com-pre-ha
|
||||
ln -s /etc/nginx/sites-available/ai.51easyai.com-pre-ha \
|
||||
/etc/nginx/sites-enabled/ai.51easyai.com
|
||||
/etc/nginx/conf.d/easyai-ai-gateway-ai.51easyai.com.conf
|
||||
nginx -t
|
||||
systemctl reload nginx
|
||||
REMOTE
|
||||
@@ -327,13 +329,14 @@ echo '[cutover] starting both application sites with River workers frozen'
|
||||
AI_GATEWAY_ACTIVATE_WORKERS=false \
|
||||
"$script_dir/bootstrap-platform.sh" activate "$release_manifest"
|
||||
|
||||
for site_ip in 10.77.0.1 10.77.0.2; do
|
||||
for site_endpoint in 10.77.0.1:31088:31178 10.77.0.2:31089:31179; do
|
||||
IFS=: read -r site_ip api_port web_port <<<"$site_endpoint"
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" \
|
||||
"curl -fsS --max-time 10 http://$site_ip:30088/api/v1/healthz >/dev/null"
|
||||
"curl -fsS --max-time 10 http://$site_ip:$api_port/api/v1/healthz >/dev/null"
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" \
|
||||
"curl -fsS --max-time 10 http://$site_ip:30088/api/v1/readyz | grep -q '\"ok\":true'"
|
||||
"curl -fsS --max-time 10 http://$site_ip:$api_port/api/v1/readyz | grep -q '\"ok\":true'"
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" \
|
||||
"curl -fsS --max-time 10 http://$site_ip:30178/ | grep -q 'EasyAI AI Gateway'"
|
||||
"curl -fsS --max-time 10 http://$site_ip:$web_port/ | grep -q 'EasyAI AI Gateway'"
|
||||
done
|
||||
|
||||
"$script_dir/install-nginx-edges.sh" activate
|
||||
|
||||
@@ -31,19 +31,42 @@ k3s kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=30
|
||||
k3s kubectl set env deployment/easyai-api-ningbo -n easyai \
|
||||
AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=false >/dev/null
|
||||
k3s kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s
|
||||
curl -fsS http://10.77.0.1:30088/api/v1/readyz | grep -q '"ok":true'
|
||||
curl -fsS http://10.77.0.2:30088/api/v1/readyz | grep -q '"ok":true'
|
||||
for _ in $(seq 1 60); do
|
||||
pods=$(k3s kubectl get pods -n easyai \
|
||||
-l app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo \
|
||||
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||
pod_count=$(wc -w <<<"$pods" | tr -d ' ')
|
||||
if [[ $pod_count == 1 ]]; then
|
||||
pod=$pods
|
||||
worker_enabled=$(k3s kubectl get pod "$pod" -n easyai \
|
||||
-o jsonpath='{.spec.containers[0].env[?(@.name=="AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED")].value}')
|
||||
pod_ready=$(k3s kubectl get pod "$pod" -n easyai \
|
||||
-o jsonpath='{.status.containerStatuses[0].ready}')
|
||||
if [[ $worker_enabled == false && $pod_ready == true ]]; then
|
||||
break
|
||||
fi
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
[[ ${pod_count:-0} == 1 && ${worker_enabled:-} == false && ${pod_ready:-} == true ]]
|
||||
k3s kubectl logs "$pod" -n easyai | grep -q 'asynchronous queue worker disabled for this process'
|
||||
curl -fsS http://10.77.0.1:31088/api/v1/readyz | grep -q '"ok":true'
|
||||
curl -fsS http://10.77.0.2:31089/api/v1/readyz | grep -q '"ok":true'
|
||||
REMOTE
|
||||
|
||||
credentials_file=$(mktemp "$cluster_root/.local-secrets/cluster/e2e.XXXXXX")
|
||||
trap 'rm -f -- "$credentials_file"; restore_workers' EXIT HUP INT TERM
|
||||
chmod 0600 "$credentials_file"
|
||||
printf 'E2E_ACCOUNT_B64=%s\n' "$(printf '%s' "$AI_GATEWAY_ONLINE_ACCOUNT" | base64 | tr -d '\n')" \
|
||||
>"$credentials_file"
|
||||
printf 'E2E_PASSWORD_B64=%s\n' "$(printf '%s' "$AI_GATEWAY_ONLINE_PASSWORD" | base64 | tr -d '\n')" \
|
||||
>>"$credentials_file"
|
||||
printf 'AI_GATEWAY_E2E_IMAGE_MODEL=%s\n' "${AI_GATEWAY_E2E_IMAGE_MODEL:-openai:gpt-image-1}" \
|
||||
>>"$credentials_file"
|
||||
{
|
||||
printf 'E2E_ACCOUNT_B64=%s\n' \
|
||||
"$(printf '%s' "$AI_GATEWAY_ONLINE_ACCOUNT" | base64 | tr -d '\n')"
|
||||
printf 'E2E_PASSWORD_B64=%s\n' \
|
||||
"$(printf '%s' "$AI_GATEWAY_ONLINE_PASSWORD" | base64 | tr -d '\n')"
|
||||
printf 'AI_GATEWAY_E2E_IMAGE_MODEL=%s\n' \
|
||||
"${AI_GATEWAY_E2E_IMAGE_MODEL:-openai:gpt-image-1}"
|
||||
printf 'AI_GATEWAY_E2E_IMAGE_SIZE=%s\n' \
|
||||
"${AI_GATEWAY_E2E_IMAGE_SIZE:-2048x2048}"
|
||||
} >"$credentials_file"
|
||||
cluster_scp "$credentials_file" "$CLUSTER_NINGBO_HOST:/root/easyai-cross-node-e2e.env"
|
||||
cluster_scp "$script_dir/cross-node-file-e2e.mjs" \
|
||||
"$CLUSTER_NINGBO_HOST:/root/easyai-cross-node-file-e2e.mjs"
|
||||
|
||||
@@ -115,14 +115,18 @@ REMOTE
|
||||
echo 'application_acceptance=PASS api=2 web=2 digests=pinned'
|
||||
|
||||
for edge_ip in "${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}"; do
|
||||
curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
|
||||
https://ai.51easyai.com/api/v1/healthz | grep -q easyai-ai-gateway
|
||||
curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
|
||||
https://ai.51easyai.com/api/v1/readyz | grep -q '"ok":true'
|
||||
curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
|
||||
https://ai.51easyai.com/api/v1/openapi.json | grep -q '"/api/v1/chat/completions"'
|
||||
curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
|
||||
https://ai.51easyai.com/ | grep -q 'EasyAI AI Gateway'
|
||||
health_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
|
||||
https://ai.51easyai.com/api/v1/healthz)
|
||||
grep -q easyai-ai-gateway <<<"$health_body"
|
||||
ready_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
|
||||
https://ai.51easyai.com/api/v1/readyz)
|
||||
grep -q '"ok":true' <<<"$ready_body"
|
||||
openapi_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
|
||||
https://ai.51easyai.com/api/v1/openapi.json)
|
||||
grep -q '"/api/v1/chat/completions"' <<<"$openapi_body"
|
||||
web_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
|
||||
https://ai.51easyai.com/)
|
||||
grep -q 'EasyAI AI Gateway' <<<"$web_body"
|
||||
done
|
||||
echo 'nginx_edges=PASS ningbo=true hongkong=true dns_changed=false'
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user