Skip to content

Commit ee8ef90

Browse files
NikolaSimsicNikolaSimsic
authored
feat: improve re-triggering (#322)
2 parents 7a4d665 + 31a67ce commit ee8ef90

1 file changed

Lines changed: 38 additions & 162 deletions

File tree

‎.github/workflows/NightlyDispatcher.yml‎

Lines changed: 38 additions & 162 deletions
Original file line numberDiff line numberDiff line change
@@ -12,97 +12,48 @@ on:
1212
- cron: '0 4 * * *'
1313

1414
jobs:
15-
dispatch-main:
16-
if: github.event.schedule == '0 22 * * *'
15+
# This job determines which branch to target based on the schedule
16+
determine-target:
1717
runs-on: ubuntu-latest
1818
outputs:
19-
run-id: ${{ steps.trigger.outputs.run_id }}
19+
target-branch: ${{ steps.set-branch.outputs.branch }}
20+
run-name: ${{ steps.set-branch.outputs.run-name }}
2021
steps:
21-
- name: "Trigger Native Pipeline on main"
22-
id: trigger
23-
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7
24-
with:
25-
script: |
26-
const response = await github.rest.actions.createWorkflowDispatch({
27-
owner: context.repo.owner,
28-
repo: context.repo.repo,
29-
workflow_id: 'NativePipeline.yml',
30-
ref: 'main',
31-
inputs: {
32-
run_name: 'Nightly Main Branch Pipeline',
33-
workspace: '*-native'
34-
}
35-
});
36-
37-
console.log('Workflow dispatch triggered successfully');
38-
39-
// Wait longer and find the correct run
40-
let runId = null;
41-
let attempts = 0;
42-
const maxAttempts = 18; // Increase to 3 minutes
43-
44-
while (!runId && attempts < maxAttempts) {
45-
await new Promise(resolve => setTimeout(resolve, 10000)); // Wait 10 seconds
46-
attempts++;
47-
console.log(`Attempt ${attempts}/${maxAttempts} to find triggered run...`);
48-
49-
try {
50-
const runs = await github.rest.actions.listWorkflowRuns({
51-
owner: context.repo.owner,
52-
repo: context.repo.repo,
53-
workflow_id: 'NativePipeline.yml',
54-
branch: 'main',
55-
per_page: 10
56-
});
57-
58-
// Find run created in the last 10 minutes
59-
const tenMinutesAgo = new Date();
60-
tenMinutesAgo.setMinutes(tenMinutesAgo.getMinutes() - 10);
61-
62-
const recentRun = runs.data.workflow_runs.find(run => {
63-
const runDate = new Date(run.created_at);
64-
return runDate > tenMinutesAgo && run.event === 'workflow_dispatch';
65-
});
66-
67-
if (recentRun) {
68-
runId = recentRun.id;
69-
console.log(`Found triggered run ID: ${runId}`);
70-
break;
71-
}
72-
} catch (error) {
73-
console.log(`Error finding run: ${error.message}`);
74-
}
75-
}
76-
77-
if (!runId) {
78-
console.log('Could not find the triggered workflow run - will continue without monitoring');
79-
core.setOutput('dispatch_success', 'false');
80-
core.setOutput('run_id', '');
81-
return;
82-
}
83-
84-
core.setOutput('dispatch_success', 'true');
85-
core.setOutput('run_id', runId);
86-
return runId;
22+
- id: set-branch
23+
run: |
24+
if [[ "${{ github.event.schedule }}" == "0 22 * * *" ]]; then
25+
echo "branch=main" >> $GITHUB_OUTPUT
26+
echo "run-name=Nightly Main Branch Pipeline" >> $GITHUB_OUTPUT
27+
else
28+
echo "branch=version/mx/10" >> $GITHUB_OUTPUT
29+
echo "run-name=Nightly version/mx/10 Branch Pipeline" >> $GITHUB_OUTPUT
30+
fi
8731
88-
dispatch-version-mx-10:
89-
if: github.event.schedule == '0 4 * * *'
32+
# This single job handles dispatching to the correct branch
33+
dispatch-pipeline:
34+
needs: determine-target
9035
runs-on: ubuntu-latest
9136
outputs:
9237
run-id: ${{ steps.trigger.outputs.run_id }}
38+
target-branch: ${{ needs.determine-target.outputs.target-branch }}
9339
steps:
94-
- name: "Trigger Native Pipeline on version/mx/10"
40+
- name: "Trigger Native Pipeline on ${{ needs.determine-target.outputs.target-branch }}"
9541
id: trigger
9642
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7
9743
with:
9844
script: |
45+
const targetBranch = '${{ needs.determine-target.outputs.target-branch }}';
46+
const runName = '${{ needs.determine-target.outputs.run-name }}';
47+
48+
console.log(`Triggering workflow on branch: ${targetBranch}`);
49+
9950
const response = await github.rest.actions.createWorkflowDispatch({
10051
owner: context.repo.owner,
10152
repo: context.repo.repo,
10253
workflow_id: 'NativePipeline.yml',
103-
ref: 'version/mx/10',
54+
ref: targetBranch,
10455
inputs: {
105-
run_name: 'Nightly version/mx/10 Branch Pipeline',
56+
run_name: runName,
10657
workspace: '*-native'
10758
}
10859
});
@@ -112,7 +63,7 @@ jobs:
11263
// Wait longer and find the correct run
11364
let runId = null;
11465
let attempts = 0;
115-
const maxAttempts = 18; // Increase to 3 minutes
66+
const maxAttempts = 18; // 3 minutes
11667
11768
while (!runId && attempts < maxAttempts) {
11869
await new Promise(resolve => setTimeout(resolve, 10000)); // Wait 10 seconds
@@ -124,7 +75,7 @@ jobs:
12475
owner: context.repo.owner,
12576
repo: context.repo.repo,
12677
workflow_id: 'NativePipeline.yml',
127-
branch: 'version/mx/10',
78+
branch: targetBranch,
12879
per_page: 10
12980
});
13081
@@ -158,109 +109,33 @@ jobs:
158109
core.setOutput('run_id', runId);
159110
return runId;
160111
161-
auto-retry-main:
162-
needs: dispatch-main
163-
if: always() && needs.dispatch-main.outputs.run-id != '' && github.event.schedule == '0 22 * * *'
164-
runs-on: ubuntu-latest
165-
env:
166-
RETRY_COUNT: 0 # Track retry attempts
167-
steps:
168-
- name: "Monitor and retry failed jobs"
169-
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7
170-
with:
171-
script: |
172-
const runId = '${{ needs.dispatch-main.outputs.run-id }}';
173-
const MAX_RETRIES = 1; // Only retry once
174-
175-
if (!runId || runId === 'null' || runId === '') {
176-
console.log('No run ID available from dispatch job - skipping monitoring');
177-
return;
178-
}
179-
180-
console.log(`Starting monitoring for run ID: ${runId}`);
181-
182-
// Poll for completion with timeout
183-
let run;
184-
let pollAttempts = 0;
185-
const maxPollAttempts = 120; // 2 hours max
186-
187-
do {
188-
if (pollAttempts >= maxPollAttempts) {
189-
console.log('Monitoring timeout reached (2 hours). Stopping.');
190-
return;
191-
}
192-
193-
await new Promise(resolve => setTimeout(resolve, 60000)); // Wait 1 minute
194-
pollAttempts++;
195-
196-
try {
197-
run = await github.rest.actions.getWorkflowRun({
198-
owner: context.repo.owner,
199-
repo: context.repo.repo,
200-
run_id: runId
201-
});
202-
console.log(`Poll #${pollAttempts}: Run status: ${run.data.status}, conclusion: ${run.data.conclusion || 'N/A'}, attempt: ${run.data.run_attempt}`);
203-
} catch (error) {
204-
console.log(`Error getting run status: ${error.message}`);
205-
continue;
206-
}
207-
} while (run.data.status === 'in_progress' || run.data.status === 'queued');
208-
209-
// Check if we should retry
210-
if (run.data.conclusion === 'failure') {
211-
console.log(`Pipeline failed on attempt ${run.data.run_attempt}. Checking if retry needed...`);
212-
213-
// Only retry if this is the first attempt
214-
if (run.data.run_attempt === 1) {
215-
console.log('Triggering retry for failed jobs...');
216-
try {
217-
await github.rest.actions.reRunWorkflowFailedJobs({
218-
owner: context.repo.owner,
219-
repo: context.repo.repo,
220-
run_id: runId
221-
});
222-
console.log('Retry triggered successfully for failed jobs only.');
223-
} catch (error) {
224-
console.log(`Failed to trigger retry: ${error.message}`);
225-
}
226-
} else {
227-
console.log('This was already a retry attempt. Not retrying again.');
228-
}
229-
} else if (run.data.conclusion === 'success') {
230-
console.log('Pipeline completed successfully!');
231-
} else {
232-
console.log(`Pipeline completed with conclusion: ${run.data.conclusion}`);
233-
}
234-
235-
auto-retry-version-mx-10:
236-
needs: dispatch-version-mx-10
237-
if: always() && needs.dispatch-version-mx-10.outputs.run-id != '' && github.event.schedule == '0 4 * * *'
112+
auto-retry:
113+
needs: [dispatch-pipeline, determine-target]
114+
if: always() && needs.dispatch-pipeline.outputs.run-id != ''
238115
runs-on: ubuntu-latest
239-
env:
240-
RETRY_COUNT: 0 # Track retry attempts
241116
steps:
242117
- name: "Monitor and retry failed jobs"
243118
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7
244119
with:
245120
script: |
246-
const runId = '${{ needs.dispatch-version-mx-10.outputs.run-id }}';
247-
const MAX_RETRIES = 1; // Only retry once
121+
const runId = '${{ needs.dispatch-pipeline.outputs.run-id }}';
122+
const targetBranch = '${{ needs.dispatch-pipeline.outputs.target-branch }}';
248123
249124
if (!runId || runId === 'null' || runId === '') {
250125
console.log('No run ID available from dispatch job - skipping monitoring');
251126
return;
252127
}
253128
254-
console.log(`Starting monitoring for run ID: ${runId}`);
129+
console.log(`Starting monitoring for run ID: ${runId} on branch ${targetBranch}`);
255130
256131
// Poll for completion with timeout
257132
let run;
258133
let pollAttempts = 0;
259-
const maxPollAttempts = 120; // 2 hours max
134+
const maxPollAttempts = 240; // 4 hours max
260135
261136
do {
262137
if (pollAttempts >= maxPollAttempts) {
263-
console.log('Monitoring timeout reached (2 hours). Stopping.');
138+
console.log('Monitoring timeout reached (4 hours). Stopping.');
264139
return;
265140
}
266141
@@ -286,7 +161,7 @@ jobs:
286161
287162
// Only retry if this is the first attempt
288163
if (run.data.run_attempt === 1) {
289-
console.log('Triggering retry for failed jobs...');
164+
console.log(`Triggering retry for failed jobs on branch ${targetBranch}...`);
290165
try {
291166
await github.rest.actions.reRunWorkflowFailedJobs({
292167
owner: context.repo.owner,
@@ -295,7 +170,8 @@ jobs:
295170
});
296171
console.log('Retry triggered successfully for failed jobs only.');
297172
} catch (error) {
298-
console.log(`Failed to trigger retry: ${error.message}`);
173+
console.error(`Failed to trigger retry: ${error.message}`);
174+
console.log('Detailed error:', JSON.stringify(error, null, 2));
299175
}
300176
} else {
301177
console.log('This was already a retry attempt. Not retrying again.');

0 commit comments

Comments
 (0)