GenarrativeAI/Genarrative
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(deploy): tune gallery load shedding for release

Browse Source
This commit is contained in:
kdletters
2026-05-19 05:03:23 +08:00
parent 05a0f34722
commit fa43410c8c
7 changed files with 27 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
deploy/container/README.md
View File

@@ -132,7 +132,7 @@ spacetime publish genarrative-loadtest --server http://127.0.0.1:13101 --module-
如果要压 1000 HTTP req/s，把 `PEAK_RPS` 调到 `500`；如果要压 5000 HTTP req/s，把 `PEAK_RPS` 调到 `2500`，并同时提高 `PREALLOCATED_VUS` / `MAX_VUS`，观察是否先被带宽、Nginx `limit_conn` / `limit_req` 或 api-server 分组背压限制。当前容器 Nginx 对公开 gallery list 使用 `genarrative_gallery_rps`，公开详情和普通 API 使用 `genarrative_api_rps`，后台 API 使用 `genarrative_admin_rps`；api-server 侧对应 `GENARRATIVE_API_GALLERY_MAX_CONCURRENT_REQUESTS`、`GENARRATIVE_API_DETAIL_MAX_CONCURRENT_REQUESTS` 和 `GENARRATIVE_API_ADMIN_MAX_CONCURRENT_REQUESTS`。
2026-05-19 的 2C / 2G 容器压测结论：公开 gallery list 的 `limit_conn=320` 与 `GENARRATIVE_API_GALLERY_MAX_CONCURRENT_REQUESTS=320` 是当前较稳的上限。用宿主机 k6 打 `http://127.0.0.1:18080`，`PEAK_RPS=1000` 等价于约 2000 HTTP req/s 的两接口组合压测；320 档无 dropped iterations、无 5xx、无 OOM，约 `151710` 个 200 与 `34310` 个 429，200 请求 `request_time p95=0.292s`。继续抬到 336 / 352 不会有效吃满 api-server CPU，反而让 200 数量减少、p95 升到约 0.31s / 0.32s，SpacetimeDB 内存尾部逼近 `880MiB / 896MiB`，下游内存先到危险区。当前不要为了降低“剩余 CPU”继续抬公开列表并发；下一步应减少成功列表请求后的 SpacetimeDB tracking 写入或优化下游状态，而不是放大入口并发。
2026-05-19 的 2C / 2G 容器压测结论：公开 gallery list 的 `limit_conn=320`、`limit_req rate=5000r/s burst=4096` 与 `GENARRATIVE_API_GALLERY_MAX_CONCURRENT_REQUESTS=320` 是当前发布口径。用宿主机 k6 打 `http://127.0.0.1:18080`，`PEAK_RPS=2500` 等价于约 5000 HTTP req/s 的两接口组合压测；连续 10 轮不重启 SpacetimeDB 的平均实际吞吐约 `4219 HTTP req/s`，总计 `1,897,357` 个 200、`212,542` 个 429、`0` 个 5xx，200 请求平均 `p95=123ms`、`p99=234ms`。该档会让 SpacetimeDB 内存从约 `366MiB` 累积到约 `885MiB / 896MiB`，下游内存先到危险区。当前不要为了降低“剩余 CPU”继续抬公开列表并发；下一步应减少成功列表请求后的 SpacetimeDB tracking 写入或优化下游连接 / 订阅状态，而不是放大入口并发。
### 内存采样

6
deploy/container/nginx.conf
View File

@@ -21,7 +21,7 @@ http {
}
limit_conn_zone $binary_remote_addr zone=genarrative_api_conn:10m;
limit_req_zone $binary_remote_addr zone=genarrative_gallery_rps:10m rate=2400r/s;
limit_req_zone $binary_remote_addr zone=genarrative_gallery_rps:10m rate=5000r/s;
limit_req_zone $binary_remote_addr zone=genarrative_api_rps:10m rate=300r/s;
limit_req_zone $binary_remote_addr zone=genarrative_admin_rps:10m rate=30r/s;
@@ -91,7 +91,7 @@ http {
location = /api/runtime/puzzle/gallery {
default_type application/json;
limit_conn genarrative_api_conn 320;
limit_req zone=genarrative_gallery_rps burst=256 nodelay;
limit_req zone=genarrative_gallery_rps burst=4096 nodelay;
proxy_pass http://genarrative_api;
proxy_http_version 1.1;
@@ -111,7 +111,7 @@ http {
location = /api/runtime/custom-world-gallery {
default_type application/json;
limit_conn genarrative_api_conn 320;
limit_req zone=genarrative_gallery_rps burst=256 nodelay;
limit_req zone=genarrative_gallery_rps burst=4096 nodelay;
proxy_pass http://genarrative_api;
proxy_http_version 1.1;

4
deploy/env/api-server.env.example vendored
View File

@@ -8,8 +8,8 @@ GENARRATIVE_API_LOG=info,tower_http=info
GENARRATIVE_API_LISTEN_BACKLOG=1024
GENARRATIVE_API_WORKER_THREADS=4
GENARRATIVE_API_MAX_CONCURRENT_REQUESTS=512
GENARRATIVE_API_GALLERY_MAX_CONCURRENT_REQUESTS=64
GENARRATIVE_API_DETAIL_MAX_CONCURRENT_REQUESTS=32
GENARRATIVE_API_GALLERY_MAX_CONCURRENT_REQUESTS=320
GENARRATIVE_API_DETAIL_MAX_CONCURRENT_REQUESTS=64
GENARRATIVE_API_ADMIN_MAX_CONCURRENT_REQUESTS=16
GENARRATIVE_TRACKING_OUTBOX_ENABLED=true
GENARRATIVE_TRACKING_OUTBOX_DIR=/var/lib/genarrative/tracking-outbox

10
deploy/nginx/genarrative-dev-http.conf
View File

@@ -14,7 +14,7 @@ upstream genarrative_api {
}
limit_conn_zone $binary_remote_addr zone=genarrative_api_conn:10m;
limit_req_zone $binary_remote_addr zone=genarrative_gallery_rps:10m rate=650r/s;
limit_req_zone $binary_remote_addr zone=genarrative_gallery_rps:10m rate=5000r/s;
limit_req_zone $binary_remote_addr zone=genarrative_api_rps:10m rate=300r/s;
limit_req_zone $binary_remote_addr zone=genarrative_admin_rps:10m rate=30r/s;
@@ -93,8 +93,8 @@ server {
location = /api/runtime/puzzle/gallery {
default_type application/json;
limit_conn genarrative_api_conn 64;
limit_req zone=genarrative_gallery_rps burst=64 nodelay;
limit_conn genarrative_api_conn 320;
limit_req zone=genarrative_gallery_rps burst=4096 nodelay;
if ($genarrative_maintenance) {
return 503 '{"ok":false,"error":{"code":"MAINTENANCE","message":"服务维护中"}}';
@@ -117,8 +117,8 @@ server {
location = /api/runtime/custom-world-gallery {
default_type application/json;
limit_conn genarrative_api_conn 64;
limit_req zone=genarrative_gallery_rps burst=64 nodelay;
limit_conn genarrative_api_conn 320;
limit_req zone=genarrative_gallery_rps burst=4096 nodelay;
if ($genarrative_maintenance) {
return 503 '{"ok":false,"error":{"code":"MAINTENANCE","message":"服务维护中"}}';

10
deploy/nginx/genarrative.conf
View File

@@ -12,7 +12,7 @@ upstream genarrative_api {
}
limit_conn_zone $binary_remote_addr zone=genarrative_api_conn:10m;
limit_req_zone $binary_remote_addr zone=genarrative_gallery_rps:10m rate=650r/s;
limit_req_zone $binary_remote_addr zone=genarrative_gallery_rps:10m rate=5000r/s;
limit_req_zone $binary_remote_addr zone=genarrative_api_rps:10m rate=300r/s;
limit_req_zone $binary_remote_addr zone=genarrative_admin_rps:10m rate=30r/s;
@@ -113,8 +113,8 @@ server {
location = /api/runtime/puzzle/gallery {
default_type application/json;
limit_conn genarrative_api_conn 64;
limit_req zone=genarrative_gallery_rps burst=64 nodelay;
limit_conn genarrative_api_conn 320;
limit_req zone=genarrative_gallery_rps burst=4096 nodelay;
if ($genarrative_maintenance) {
return 503 '{"ok":false,"error":{"code":"MAINTENANCE","message":"服务维护中"}}';
@@ -137,8 +137,8 @@ server {
location = /api/runtime/custom-world-gallery {
default_type application/json;
limit_conn genarrative_api_conn 64;
limit_req zone=genarrative_gallery_rps burst=64 nodelay;
limit_conn genarrative_api_conn 320;
limit_req zone=genarrative_gallery_rps burst=4096 nodelay;
if ($genarrative_maintenance) {
return 503 '{"ok":false,"error":{"code":"MAINTENANCE","message":"服务维护中"}}';