diff --git a/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml b/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml index 6e68c036f1..6c1f64f45e 100644 --- a/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml +++ b/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml @@ -16,7 +16,7 @@ env: jobs: restart-dev-and-test-with-custom-base-port: - runs-on: ubicloud-standard-8 + runs-on: ubicloud-standard-16 env: NEXT_PUBLIC_STACK_PORT_PREFIX: "69" diff --git a/.github/workflows/restart-dev-and-test.yaml b/.github/workflows/restart-dev-and-test.yaml index 7d24e87b63..831148e4e7 100644 --- a/.github/workflows/restart-dev-and-test.yaml +++ b/.github/workflows/restart-dev-and-test.yaml @@ -16,7 +16,7 @@ env: jobs: restart-dev-and-test: - runs-on: ubicloud-standard-8 + runs-on: ubicloud-standard-16 steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/setup-tests-with-custom-base-port.yaml b/.github/workflows/setup-tests-with-custom-base-port.yaml index a4a008dc9f..b3d15e503f 100644 --- a/.github/workflows/setup-tests-with-custom-base-port.yaml +++ b/.github/workflows/setup-tests-with-custom-base-port.yaml @@ -16,7 +16,7 @@ env: jobs: setup-tests-with-custom-base-port: - runs-on: ubicloud-standard-8 + runs-on: ubicloud-standard-16 env: NEXT_PUBLIC_STACK_PORT_PREFIX: "69" diff --git a/.github/workflows/setup-tests.yaml b/.github/workflows/setup-tests.yaml index 9efcf6c8fe..d20748c93e 100644 --- a/.github/workflows/setup-tests.yaml +++ b/.github/workflows/setup-tests.yaml @@ -16,7 +16,7 @@ env: jobs: setup-tests: - runs-on: ubicloud-standard-8 + runs-on: ubicloud-standard-16 steps: - uses: actions/checkout@v6 diff --git a/apps/backend/src/lib/clickhouse.tsx b/apps/backend/src/lib/clickhouse.tsx index f4e62daf54..e3045d69de 100644 --- a/apps/backend/src/lib/clickhouse.tsx +++ b/apps/backend/src/lib/clickhouse.tsx @@ -48,30 +48,44 @@ export const getQueryTimingStats = async (client: ClickHouseClient, queryId: str password: clickhouseAdminPassword, }, }); - const profile = await client.query({ - query: ` - SELECT - ProfileEvents['CPUTimeMicroseconds'] / 1000 AS cpu_time_ms, - ProfileEvents['RealTimeMicroseconds'] / 1000 AS wall_clock_time_ms - FROM system.query_log - WHERE query_id = {query_id:String} AND type = 'QueryFinish' - ORDER BY event_time DESC - LIMIT 1 - `, - query_params: { query_id: queryId }, - auth: { - username: clickhouseAdminUser, - password: clickhouseAdminPassword, - }, - format: "JSON", - }); + const queryProfile = async () => { + const profile = await client.query({ + query: ` + SELECT + ProfileEvents['CPUTimeMicroseconds'] / 1000 AS cpu_time_ms, + ProfileEvents['RealTimeMicroseconds'] / 1000 AS wall_clock_time_ms + FROM system.query_log + WHERE query_id = {query_id:String} AND type = 'QueryFinish' + ORDER BY event_time DESC + LIMIT 1 + `, + query_params: { query_id: queryId }, + auth: { + username: clickhouseAdminUser, + password: clickhouseAdminPassword, + }, + format: "JSON", + }); + + return await profile.json<{ + cpu_time_ms: number, + wall_clock_time_ms: number, + }>(); + }; - const stats = await profile.json<{ - cpu_time_ms: number, - wall_clock_time_ms: number, - }>(); - if (stats.data.length !== 1) { - throw new StackAssertionError(`Unexpected number of query log results: ${stats.data.length}`, { data: stats.data }); + const retryDelaysMs = [75, 150, 300, 600, 1200]; + for (let attempt = 0; attempt <= retryDelaysMs.length; attempt += 1) { + const stats = await queryProfile(); + if (stats.data.length === 1) { + return stats.data[0]; + } + if (stats.data.length > 1) { + throw new StackAssertionError(`Unexpected number of query log results: ${stats.data.length}`, { data: stats.data }); + } + if (attempt < retryDelaysMs.length) { + await new Promise((resolve) => setTimeout(resolve, retryDelaysMs[attempt])); + } } - return stats.data[0]; + + throw new StackAssertionError("Unexpected number of query log results: 0", { data: [] }); };