Skip to content

Commit 77773ad

Browse files
igchorigchor
authored
[SYCL][UR][L0 v2] Move all enqueue-related logic to command_list_manager (#18837)
Almost all in_order_queue functions are implemented by just forwarding the call to the command_list_manager. This is needed to implement out-of-order queue without duplicating the implementation.
1 parent 5539f67 commit 77773ad

12 files changed

Lines changed: 1188 additions & 1315 deletions

File tree

‎unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp‎

Lines changed: 63 additions & 132 deletions
Original file line numberDiff line numberDiff line change
@@ -70,10 +70,11 @@ ur_exp_command_buffer_handle_t_::ur_exp_command_buffer_handle_t_(
7070
isInOrder(desc ? desc->isInOrder : false),
7171
commandListManager(
7272
context, device,
73-
std::forward<v2::raii::command_list_unique_handle>(commandList),
74-
isInOrder ? v2::EVENT_FLAGS_COUNTER : 0, nullptr,
75-
PoolCacheType::Regular),
76-
context(context), device(device) {}
73+
std::forward<v2::raii::command_list_unique_handle>(commandList)),
74+
context(context), device(device),
75+
eventPool(context->getEventPoolCache(PoolCacheType::Regular)
76+
.borrow(device->Id.value(),
77+
isInOrder ? v2::EVENT_FLAGS_COUNTER : 0)) {}
7778

7879
ur_exp_command_buffer_sync_point_t
7980
ur_exp_command_buffer_handle_t_::getSyncPoint(ur_event_handle_t event) {
@@ -155,7 +156,6 @@ ur_result_t ur_exp_command_buffer_handle_t_::registerExecutionEventUnlocked(
155156
}
156157
if (nextExecutionEvent) {
157158
currentExecution = nextExecutionEvent;
158-
UR_CALL(nextExecutionEvent->retain());
159159
}
160160
return UR_RESULT_SUCCESS;
161161
}
@@ -202,6 +202,21 @@ ur_result_t ur_exp_command_buffer_handle_t_::applyUpdateCommands(
202202

203203
return UR_RESULT_SUCCESS;
204204
}
205+
206+
ur_event_handle_t ur_exp_command_buffer_handle_t_::createEventIfRequested(
207+
ur_exp_command_buffer_sync_point_t *retSyncPoint) {
208+
if (retSyncPoint == nullptr) {
209+
return nullptr;
210+
}
211+
212+
auto event = eventPool->allocate();
213+
event->setQueue(nullptr);
214+
215+
*retSyncPoint = getSyncPoint(event);
216+
217+
return event;
218+
}
219+
205220
namespace ur::level_zero {
206221

207222
ur_result_t
@@ -292,18 +307,11 @@ ur_result_t urCommandBufferAppendKernelLaunchExp(
292307
}
293308
auto eventsWaitList = commandBuffer->getWaitListFromSyncPoints(
294309
syncPointWaitList, numSyncPointsInWaitList);
295-
ur_event_handle_t *event = nullptr;
296-
ur_event_handle_t signalEvent = nullptr;
297-
if (retSyncPoint != nullptr) {
298-
event = &signalEvent;
299-
}
300-
UR_CALL(commandListLocked->appendKernelLaunch(
301-
hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, pLocalWorkSize,
302-
numSyncPointsInWaitList, eventsWaitList, event));
303310

304-
if (retSyncPoint != nullptr) {
305-
*retSyncPoint = commandBuffer->getSyncPoint(signalEvent);
306-
}
311+
UR_CALL(commandListLocked->appendKernelLaunch(
312+
hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, pLocalWorkSize, 0,
313+
nullptr, numSyncPointsInWaitList, eventsWaitList,
314+
commandBuffer->createEventIfRequested(retSyncPoint)));
307315

308316
return UR_RESULT_SUCCESS;
309317
} catch (...) {
@@ -324,17 +332,11 @@ ur_result_t urCommandBufferAppendUSMMemcpyExp(
324332
auto commandListLocked = hCommandBuffer->commandListManager.lock();
325333
auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
326334
pSyncPointWaitList, numSyncPointsInWaitList);
327-
ur_event_handle_t *event = nullptr;
328-
ur_event_handle_t signalEvent = nullptr;
329-
if (pSyncPoint != nullptr) {
330-
event = &signalEvent;
331-
}
335+
332336
UR_CALL(commandListLocked->appendUSMMemcpy(
333-
false, pDst, pSrc, size, numSyncPointsInWaitList, eventsWaitList, event));
337+
false, pDst, pSrc, size, numSyncPointsInWaitList, eventsWaitList,
338+
hCommandBuffer->createEventIfRequested(pSyncPoint)));
334339

335-
if (pSyncPoint != nullptr) {
336-
*pSyncPoint = hCommandBuffer->getSyncPoint(signalEvent);
337-
}
338340
return UR_RESULT_SUCCESS;
339341
} catch (...) {
340342
return exceptionToResult(std::current_exception());
@@ -357,18 +359,11 @@ ur_result_t urCommandBufferAppendMemBufferCopyExp(
357359
auto commandListLocked = hCommandBuffer->commandListManager.lock();
358360
auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
359361
pSyncPointWaitList, numSyncPointsInWaitList);
360-
ur_event_handle_t *event = nullptr;
361-
ur_event_handle_t signalEvent = nullptr;
362-
if (pSyncPoint != nullptr) {
363-
event = &signalEvent;
364-
}
362+
365363
UR_CALL(commandListLocked->appendMemBufferCopy(
366364
hSrcMem, hDstMem, srcOffset, dstOffset, size, numSyncPointsInWaitList,
367-
eventsWaitList, event));
365+
eventsWaitList, hCommandBuffer->createEventIfRequested(pSyncPoint)));
368366

369-
if (pSyncPoint != nullptr) {
370-
*pSyncPoint = hCommandBuffer->getSyncPoint(signalEvent);
371-
}
372367
return UR_RESULT_SUCCESS;
373368
} catch (...) {
374369
return exceptionToResult(std::current_exception());
@@ -391,18 +386,11 @@ ur_result_t urCommandBufferAppendMemBufferWriteExp(
391386
auto commandListLocked = hCommandBuffer->commandListManager.lock();
392387
auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
393388
pSyncPointWaitList, numSyncPointsInWaitList);
394-
ur_event_handle_t *event = nullptr;
395-
ur_event_handle_t signalEvent = nullptr;
396-
if (pSyncPoint != nullptr) {
397-
event = &signalEvent;
398-
}
399-
UR_CALL(commandListLocked->appendMemBufferWrite(hBuffer, false, offset, size,
400-
pSrc, numSyncPointsInWaitList,
401-
eventsWaitList, event));
402389

403-
if (pSyncPoint != nullptr) {
404-
*pSyncPoint = hCommandBuffer->getSyncPoint(signalEvent);
405-
}
390+
UR_CALL(commandListLocked->appendMemBufferWrite(
391+
hBuffer, false, offset, size, pSrc, numSyncPointsInWaitList,
392+
eventsWaitList, hCommandBuffer->createEventIfRequested(pSyncPoint)));
393+
406394
return UR_RESULT_SUCCESS;
407395
} catch (...) {
408396
return exceptionToResult(std::current_exception());
@@ -423,18 +411,11 @@ ur_result_t urCommandBufferAppendMemBufferReadExp(
423411
auto commandListLocked = hCommandBuffer->commandListManager.lock();
424412
auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
425413
pSyncPointWaitList, numSyncPointsInWaitList);
426-
ur_event_handle_t *event = nullptr;
427-
ur_event_handle_t signalEvent = nullptr;
428-
if (pSyncPoint != nullptr) {
429-
event = &signalEvent;
430-
}
431-
UR_CALL(commandListLocked->appendMemBufferRead(hBuffer, false, offset, size,
432-
pDst, numSyncPointsInWaitList,
433-
eventsWaitList, event));
434414

435-
if (pSyncPoint != nullptr) {
436-
*pSyncPoint = hCommandBuffer->getSyncPoint(signalEvent);
437-
}
415+
UR_CALL(commandListLocked->appendMemBufferRead(
416+
hBuffer, false, offset, size, pDst, numSyncPointsInWaitList,
417+
eventsWaitList, hCommandBuffer->createEventIfRequested(pSyncPoint)));
418+
438419
return UR_RESULT_SUCCESS;
439420
} catch (...) {
440421
return exceptionToResult(std::current_exception());
@@ -459,19 +440,12 @@ ur_result_t urCommandBufferAppendMemBufferCopyRectExp(
459440
auto commandListLocked = hCommandBuffer->commandListManager.lock();
460441
auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
461442
pSyncPointWaitList, numSyncPointsInWaitList);
462-
ur_event_handle_t *event = nullptr;
463-
ur_event_handle_t signalEvent = nullptr;
464-
if (pSyncPoint != nullptr) {
465-
event = &signalEvent;
466-
}
443+
467444
UR_CALL(commandListLocked->appendMemBufferCopyRect(
468445
hSrcMem, hDstMem, srcOrigin, dstOrigin, region, srcRowPitch,
469446
srcSlicePitch, dstRowPitch, dstSlicePitch, numSyncPointsInWaitList,
470-
eventsWaitList, event));
447+
eventsWaitList, hCommandBuffer->createEventIfRequested(pSyncPoint)));
471448

472-
if (pSyncPoint != nullptr) {
473-
*pSyncPoint = hCommandBuffer->getSyncPoint(signalEvent);
474-
}
475449
return UR_RESULT_SUCCESS;
476450
} catch (...) {
477451
return exceptionToResult(std::current_exception());
@@ -496,19 +470,13 @@ ur_result_t urCommandBufferAppendMemBufferWriteRectExp(
496470
auto commandListLocked = hCommandBuffer->commandListManager.lock();
497471
auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
498472
pSyncPointWaitList, numSyncPointsInWaitList);
499-
ur_event_handle_t *event = nullptr;
500-
ur_event_handle_t signalEvent = nullptr;
501-
if (pSyncPoint != nullptr) {
502-
event = &signalEvent;
503-
}
473+
504474
UR_CALL(commandListLocked->appendMemBufferWriteRect(
505475
hBuffer, false, bufferOffset, hostOffset, region, bufferRowPitch,
506476
bufferSlicePitch, hostRowPitch, hostSlicePitch, pSrc,
507-
numSyncPointsInWaitList, eventsWaitList, event));
477+
numSyncPointsInWaitList, eventsWaitList,
478+
hCommandBuffer->createEventIfRequested(pSyncPoint)));
508479

509-
if (pSyncPoint != nullptr) {
510-
*pSyncPoint = hCommandBuffer->getSyncPoint(signalEvent);
511-
}
512480
return UR_RESULT_SUCCESS;
513481
} catch (...) {
514482
return exceptionToResult(std::current_exception());
@@ -533,19 +501,13 @@ ur_result_t urCommandBufferAppendMemBufferReadRectExp(
533501
auto commandListLocked = hCommandBuffer->commandListManager.lock();
534502
auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
535503
pSyncPointWaitList, numSyncPointsInWaitList);
536-
ur_event_handle_t *event = nullptr;
537-
ur_event_handle_t signalEvent = nullptr;
538-
if (pSyncPoint != nullptr) {
539-
event = &signalEvent;
540-
}
504+
541505
UR_CALL(commandListLocked->appendMemBufferReadRect(
542506
hBuffer, false, bufferOffset, hostOffset, region, bufferRowPitch,
543507
bufferSlicePitch, hostRowPitch, hostSlicePitch, pDst,
544-
numSyncPointsInWaitList, eventsWaitList, event));
508+
numSyncPointsInWaitList, eventsWaitList,
509+
hCommandBuffer->createEventIfRequested(pSyncPoint)));
545510

546-
if (pSyncPoint != nullptr) {
547-
*pSyncPoint = hCommandBuffer->getSyncPoint(signalEvent);
548-
}
549511
return UR_RESULT_SUCCESS;
550512
} catch (...) {
551513
return exceptionToResult(std::current_exception());
@@ -565,17 +527,10 @@ ur_result_t urCommandBufferAppendUSMFillExp(
565527
auto commandListLocked = hCommandBuffer->commandListManager.lock();
566528
auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
567529
pSyncPointWaitList, numSyncPointsInWaitList);
568-
ur_event_handle_t *event = nullptr;
569-
ur_event_handle_t signalEvent = nullptr;
570-
if (pSyncPoint != nullptr) {
571-
event = &signalEvent;
572-
}
573-
UR_CALL(commandListLocked->appendUSMFill(pMemory, patternSize, pPattern, size,
574-
numSyncPointsInWaitList,
575-
eventsWaitList, event));
576-
if (pSyncPoint != nullptr) {
577-
*pSyncPoint = hCommandBuffer->getSyncPoint(signalEvent);
578-
}
530+
531+
UR_CALL(commandListLocked->appendUSMFill(
532+
pMemory, patternSize, pPattern, size, numSyncPointsInWaitList,
533+
eventsWaitList, hCommandBuffer->createEventIfRequested(pSyncPoint)));
579534
return UR_RESULT_SUCCESS;
580535
} catch (...) {
581536
return exceptionToResult(std::current_exception());
@@ -596,17 +551,11 @@ ur_result_t urCommandBufferAppendMemBufferFillExp(
596551
auto commandListLocked = hCommandBuffer->commandListManager.lock();
597552
auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
598553
pSyncPointWaitList, numSyncPointsInWaitList);
599-
ur_event_handle_t *event = nullptr;
600-
ur_event_handle_t signalEvent = nullptr;
601-
if (pSyncPoint != nullptr) {
602-
event = &signalEvent;
603-
}
554+
604555
UR_CALL(commandListLocked->appendMemBufferFill(
605556
hBuffer, pPattern, patternSize, offset, size, numSyncPointsInWaitList,
606-
eventsWaitList, event));
607-
if (pSyncPoint != nullptr) {
608-
*pSyncPoint = hCommandBuffer->getSyncPoint(signalEvent);
609-
}
557+
eventsWaitList, hCommandBuffer->createEventIfRequested(pSyncPoint)));
558+
610559
return UR_RESULT_SUCCESS;
611560
} catch (...) {
612561
return exceptionToResult(std::current_exception());
@@ -628,17 +577,11 @@ ur_result_t urCommandBufferAppendUSMPrefetchExp(
628577
auto commandListLocked = hCommandBuffer->commandListManager.lock();
629578
auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
630579
pSyncPointWaitList, numSyncPointsInWaitList);
631-
ur_event_handle_t *event = nullptr;
632-
ur_event_handle_t signalEvent = nullptr;
633-
if (pSyncPoint != nullptr) {
634-
event = &signalEvent;
635-
}
580+
636581
UR_CALL(commandListLocked->appendUSMPrefetch(
637-
pMemory, size, flags, numSyncPointsInWaitList, eventsWaitList, event));
582+
pMemory, size, flags, numSyncPointsInWaitList, eventsWaitList,
583+
hCommandBuffer->createEventIfRequested(pSyncPoint)));
638584

639-
if (pSyncPoint != nullptr) {
640-
*pSyncPoint = hCommandBuffer->getSyncPoint(signalEvent);
641-
}
642585
return UR_RESULT_SUCCESS;
643586
} catch (...) {
644587
return exceptionToResult(std::current_exception());
@@ -658,17 +601,11 @@ ur_result_t urCommandBufferAppendUSMAdviseExp(
658601
auto commandListLocked = hCommandBuffer->commandListManager.lock();
659602
auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
660603
pSyncPointWaitList, numSyncPointsInWaitList);
661-
ur_event_handle_t *event = nullptr;
662-
ur_event_handle_t signalEvent = nullptr;
663-
if (pSyncPoint != nullptr) {
664-
event = &signalEvent;
665-
}
604+
666605
UR_CALL(commandListLocked->appendUSMAdvise(
667-
pMemory, size, advice, numSyncPointsInWaitList, eventsWaitList, event));
606+
pMemory, size, advice, numSyncPointsInWaitList, eventsWaitList,
607+
hCommandBuffer->createEventIfRequested(pSyncPoint)));
668608

669-
if (pSyncPoint != nullptr) {
670-
*pSyncPoint = hCommandBuffer->getSyncPoint(signalEvent);
671-
}
672609
return UR_RESULT_SUCCESS;
673610
} catch (...) {
674611
return exceptionToResult(std::current_exception());
@@ -714,23 +651,17 @@ ur_result_t urCommandBufferAppendNativeCommandExp(
714651
auto commandListLocked = hCommandBuffer->commandListManager.lock();
715652
auto eventsWaitList = hCommandBuffer->getWaitListFromSyncPoints(
716653
pSyncPointWaitList, numSyncPointsInWaitList);
717-
ur_event_handle_t *event = nullptr;
718-
ur_event_handle_t signalEvent = nullptr;
719-
if (pSyncPoint != nullptr) {
720-
event = &signalEvent;
721-
}
722-
UR_CALL(commandListLocked->appendBarrier(numSyncPointsInWaitList,
723-
eventsWaitList, nullptr));
654+
655+
UR_CALL(commandListLocked->appendEventsWaitWithBarrier(
656+
numSyncPointsInWaitList, eventsWaitList, nullptr));
724657

725658
// Call user-defined function immediately
726659
pfnNativeCommand(pData);
727660

728661
// Barrier on all commands after user defined commands.
729-
UR_CALL(commandListLocked->appendBarrier(0, nullptr, event));
662+
UR_CALL(commandListLocked->appendEventsWaitWithBarrier(
663+
0, nullptr, hCommandBuffer->createEventIfRequested(pSyncPoint)));
730664

731-
if (pSyncPoint != nullptr) {
732-
*pSyncPoint = hCommandBuffer->getSyncPoint(signalEvent);
733-
}
734665
return UR_RESULT_SUCCESS;
735666
}
736667

‎unified-runtime/source/adapters/level_zero/v2/command_buffer.hpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ struct ur_exp_command_buffer_handle_t_ : public ur_object {
5959
const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList,
6060
uint32_t numSyncPointsInWaitList);
6161

62+
ur_event_handle_t
63+
createEventIfRequested(ur_exp_command_buffer_sync_point_t *retSyncPoint);
64+
6265
private:
6366
// Stores all sync points that are created by the command buffer.
6467
std::vector<ur_event_handle_t> syncPoints;
@@ -77,4 +80,6 @@ struct ur_exp_command_buffer_handle_t_ : public ur_object {
7780
bool isFinalized = false;
7881

7982
ur_event_handle_t currentExecution = nullptr;
83+
84+
v2::raii::cache_borrowed_event_pool eventPool;
8085
};

0 commit comments

Comments
 (0)