diff --git a/deploy/kubernetes/local-acceptance/cluster-dns.yaml b/deploy/kubernetes/local-acceptance/cluster-dns.yaml new file mode 100644 index 0000000..1782e76 --- /dev/null +++ b/deploy/kubernetes/local-acceptance/cluster-dns.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns-custom + namespace: kube-system +data: + gateway.override: | + rewrite name exact gateway.easyai.local easyai-acceptance-edge-ningbo.easyai.svc.cluster.local diff --git a/deploy/kubernetes/local-acceptance/support-services.yaml b/deploy/kubernetes/local-acceptance/support-services.yaml index 7103f72..225eabb 100644 --- a/deploy/kubernetes/local-acceptance/support-services.yaml +++ b/deploy/kubernetes/local-acceptance/support-services.yaml @@ -162,6 +162,7 @@ spec: cpu: 250m memory: 128Mi securityContext: + runAsUser: 0 allowPrivilegeEscalation: false capabilities: add: ["NET_ADMIN"] @@ -226,6 +227,7 @@ spec: cpu: 250m memory: 128Mi securityContext: + runAsUser: 0 allowPrivilegeEscalation: false capabilities: add: ["NET_ADMIN"] @@ -291,6 +293,7 @@ spec: cpu: 250m memory: 128Mi securityContext: + runAsUser: 0 allowPrivilegeEscalation: false capabilities: add: ["NET_ADMIN"] diff --git a/deploy/kubernetes/local-acceptance/tls-edges.yaml b/deploy/kubernetes/local-acceptance/tls-edges.yaml index 1bb3203..761da8d 100644 --- a/deploy/kubernetes/local-acceptance/tls-edges.yaml +++ b/deploy/kubernetes/local-acceptance/tls-edges.yaml @@ -92,7 +92,7 @@ spec: easyai.io/site: ningbo ports: - name: https - port: 8443 + port: 443 targetPort: https nodePort: 31088 --- @@ -167,6 +167,6 @@ spec: easyai.io/site: hongkong ports: - name: https - port: 8443 + port: 443 targetPort: https nodePort: 31089 diff --git a/scripts/acceptance/local-cluster.sh b/scripts/acceptance/local-cluster.sh index 6af731b..f806cf2 100755 --- a/scripts/acceptance/local-cluster.sh +++ b/scripts/acceptance/local-cluster.sh @@ -194,6 +194,23 @@ wait_rollout() { kubectl --context "$context" -n "$namespace" rollout status "$1" --timeout="${2:-10m}" } +wait_internal_gateway() { + local pod attempt=0 + pod=$(kubectl --context "$context" -n "$namespace" get pod \ + -l 'app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo' \ + -o 'jsonpath={.items[0].metadata.name}') + until kubectl --context "$context" -n "$namespace" exec "$pod" -c api -- \ + wget -q --spider https://gateway.easyai.local/api/v1/healthz \ + >/dev/null 2>&1; do + attempt=$((attempt + 1)) + if ((attempt >= 60)); then + echo 'local acceptance gateway DNS or TLS trust did not become ready' >&2 + return 1 + fi + sleep 1 + done +} + apply_runtime_secrets() { # shellcheck source=/dev/null source "$state_root/local.env" @@ -228,6 +245,9 @@ apply_runtime_secrets() { kubectl --context "$context" -n "$namespace" create secret tls easyai-acceptance-tls \ --cert="$state_root/tls.crt" --key="$state_root/tls.key" \ --dry-run=client -o yaml | kubectl --context "$context" apply -f - >/dev/null + kubectl --context "$context" -n "$namespace" create configmap easyai-acceptance-ca \ + --from-file=ca.crt="$state_root/ca.crt" --dry-run=client -o yaml | + kubectl --context "$context" apply -f - >/dev/null } run_migrations() { @@ -363,6 +383,20 @@ render_and_apply_application() { kubectl --context "$context" -n "$namespace" patch deployment "$workload" --type=json \ -p='[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"application-data","hostPath":{"path":"/var/lib/easyai-acceptance/media","type":"DirectoryOrCreate"}}}]' \ >/dev/null + local container=worker ca_patch + [[ $workload == easyai-api-* ]] && container=api + ca_patch=$(jq -cn --arg container "$container" '{ + spec:{template:{spec:{ + containers:[{ + name:$container, + env:[{name:"SSL_CERT_FILE",value:"/etc/easyai-acceptance-ca/ca.crt"}], + volumeMounts:[{name:"acceptance-ca",mountPath:"/etc/easyai-acceptance-ca",readOnly:true}] + }], + volumes:[{name:"acceptance-ca",configMap:{name:"easyai-acceptance-ca",defaultMode:292}}] + }}} + }') + kubectl --context "$context" -n "$namespace" patch deployment "$workload" \ + --type=strategic -p="$ca_patch" >/dev/null done kubectl --context "$context" -n "$namespace" scale \ deployment/easyai-web-ningbo deployment/easyai-web-hongkong --replicas=0 >/dev/null @@ -507,6 +541,7 @@ up_cluster() { config="$state_root/k3d.rendered.yaml" render_k3d_config "$config" EASYAI_ACCEPTANCE_MEDIA_DIR="$media_root" "$k3d" cluster create --config "$config" + kubectl --context "$context" apply -f "$manifest_root/cluster-dns.yaml" >/dev/null docker update --cpus 4 --memory 8g --memory-swap 8g "k3d-${cluster_name}-server-0" >/dev/null docker update --cpus 4 --memory 8g --memory-swap 8g "k3d-${cluster_name}-server-1" >/dev/null docker update --cpus 2 --memory 4g --memory-swap 4g "k3d-${cluster_name}-server-2" >/dev/null @@ -549,6 +584,7 @@ up_cluster() { wait_rollout deployment/easyai-acceptance-edge-ningbo wait_rollout deployment/easyai-acceptance-edge-hongkong bootstrap_runtime "$api_image" "$api_digest" "$snapshot" + wait_internal_gateway "$script_dir/network-fault.sh" baseline echo "local_acceptance_cluster=PASS context=$context gateways=https://127.0.0.1:18443,https://127.0.0.1:19443 ca_file=$state_root/ca.crt" }