Actual source code: device.cxx

  1: #include "petscdevice_interface_internal.hpp" /*I <petscdevice.h> I*/
  2: #include <petsc/private/petscadvancedmacros.h>

  4: #include "../impls/host/hostdevice.hpp"
  5: #include "../impls/cupm/cupmdevice.hpp"
  6: #include "../impls/sycl/sycldevice.hpp"

  8: #include <limits>  // std::numeric_limits
  9: #include <utility> // std::make_pair

 11: using namespace Petsc::device;

 13: /*
 14:   note to anyone adding more classes, the name must be ALL_CAPS_SHORT_NAME + Device exactly to
 15:   be picked up by the switch-case macros below
 16: */
 17: static host::Device HOSTDevice{PetscDeviceContextCreate_HOST};
 18: #if PetscDefined(HAVE_CUDA)
 19: static cupm::Device<cupm::DeviceType::CUDA> CUDADevice{PetscDeviceContextCreate_CUDA};
 20: #endif
 21: #if PetscDefined(HAVE_HIP)
 22: static cupm::Device<cupm::DeviceType::HIP> HIPDevice{PetscDeviceContextCreate_HIP};
 23: #endif
 24: #if PetscDefined(HAVE_SYCL)
 25: static sycl::Device SYCLDevice{PetscDeviceContextCreate_SYCL};
 26: #endif

 28: #define PETSC_DEVICE_CASE(IMPLS, func, ...) \
 29:   case PetscConcat_(PETSC_DEVICE_, IMPLS): { \
 30:     PetscConcat_(IMPLS, Device).func(__VA_ARGS__); \
 31:   } break

 33: /*
 34:   Suppose you have:

 36:   CUDADevice.myFunction(arg1,arg2)

 38:   that you would like to conditionally define and call in a switch-case:

 40:   switch(PetscDeviceType) {
 41:   #if PetscDefined(HAVE_CUDA)
 42:   case PETSC_DEVICE_CUDA: {
 43:     CUDADevice.myFunction(arg1,arg2);
 44:   } break;
 45:   #endif
 46:   }

 48:   then calling this macro:

 50:   PETSC_DEVICE_CASE_IF_PETSC_DEFINED(CUDA,myFunction,arg1,arg2)

 52:   will expand to the following case statement:

 54:   case PETSC_DEVICE_CUDA: {
 55:     CUDADevice.myFunction(arg1,arg2);
 56:   } break

 58:   if PetscDefined(HAVE_CUDA) evaluates to 1, and expand to nothing otherwise
 59: */
 60: #define PETSC_DEVICE_CASE_IF_PETSC_DEFINED(IMPLS, func, ...) PetscIfPetscDefined(PetscConcat_(HAVE_, IMPLS), PETSC_DEVICE_CASE, PetscExpandToNothing)(IMPLS, func, __VA_ARGS__)

 62: /*@C
 63:   PetscDeviceCreate - Get a new handle for a particular device (often a GPU) type

 65:   Not Collective

 67:   Input Parameters:
 68: + type  - The type of `PetscDevice`
 69: - devid - The numeric ID# of the device (pass `PETSC_DECIDE` to assign automatically)

 71:   Output Parameter:
 72: . device - The `PetscDevice`

 74:   Notes:
 75:   This routine may initialize `PetscDevice`. If this is the case, it may cause some sort of
 76:   device synchronization.

 78:   `devid` is what you might pass to `cudaSetDevice()` for example.

 80:   Level: beginner

 82: .seealso: `PetscDevice`, `PetscDeviceInitType`,
 83: `PetscDeviceInitialize()`, `PetscDeviceInitialized()`, `PetscDeviceConfigure()`,
 84: `PetscDeviceView()`, `PetscDeviceDestroy()`
 85: @*/
 86: PetscErrorCode PetscDeviceCreate(PetscDeviceType type, PetscInt devid, PetscDevice *device)
 87: {
 88:   static PetscInt PetscDeviceCounter = 0;

 92:   PetscDeviceInitializePackage();
 93:   PetscNew(device);
 94:   (*device)->id     = PetscDeviceCounter++;
 95:   (*device)->type   = type;
 96:   (*device)->refcnt = 1;
 97:   /*
 98:     if you are adding a device, you also need to add it's initialization in
 99:     PetscDeviceInitializeTypeFromOptions_Private() below
100:   */
101:   switch (type) {
102:     PETSC_DEVICE_CASE_IF_PETSC_DEFINED(HOST, getDevice, *device, devid);
103:     PETSC_DEVICE_CASE_IF_PETSC_DEFINED(CUDA, getDevice, *device, devid);
104:     PETSC_DEVICE_CASE_IF_PETSC_DEFINED(HIP, getDevice, *device, devid);
105:     PETSC_DEVICE_CASE_IF_PETSC_DEFINED(SYCL, getDevice, *device, devid);
106:   default:
107:     /* in case the above macros expand to nothing this silences any unused variable warnings */
108:     (void)(devid);
109:     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "PETSc was seemingly configured for PetscDeviceType %s but we've fallen through all cases in a switch", PetscDeviceTypes[type]);
110:   }
111:   return 0;
112: }

114: /*@C
115:   PetscDeviceDestroy - Free a `PetscDevice`

117:   Not Collective

119:   Input Parameter:
120: . device - The `PetscDevice`

122:   Level: beginner

124: .seealso: `PetscDevice`, `PetscDeviceCreate()`, `PetscDeviceConfigure()`, `PetscDeviceView()`,
125: `PetscDeviceGetType()`, `PetscDeviceGetDeviceId()`
126: @*/
127: PetscErrorCode PetscDeviceDestroy(PetscDevice *device)
128: {
130:   if (!*device) return 0;
132:   PetscDeviceDereference_Internal(*device);
133:   if ((*device)->refcnt) {
134:     *device = nullptr;
135:     return 0;
136:   }
137:   PetscFree((*device)->data);
138:   PetscFree(*device);
139:   return 0;
140: }

142: /*@C
143:   PetscDeviceConfigure - Configure a particular `PetscDevice`

145:   Not Collective

147:   Input Parameter:
148: . device - The `PetscDevice` to configure

150:   Notes:
151:   The user should not assume that this is a cheap operation.

153:   Level: beginner

155: .seealso: `PetscDevice`, `PetscDeviceCreate()`, `PetscDeviceView()`, `PetscDeviceDestroy()`,
156: `PetscDeviceGetType()`, `PetscDeviceGetDeviceId()`
157: @*/
158: PetscErrorCode PetscDeviceConfigure(PetscDevice device)
159: {
161:   /*
162:     if no available configuration is available, this cascades all the way down to default
163:     and error
164:   */
165:   switch (const auto dtype = device->type) {
166:   case PETSC_DEVICE_HOST:
167:     if (PetscDefined(HAVE_HOST)) break; // always true
168:   case PETSC_DEVICE_CUDA:
169:     if (PetscDefined(HAVE_CUDA)) break;
170:     goto error;
171:   case PETSC_DEVICE_HIP:
172:     if (PetscDefined(HAVE_HIP)) break;
173:     goto error;
174:   case PETSC_DEVICE_SYCL:
175:     if (PetscDefined(HAVE_SYCL)) break;
176:     goto error;
177:   default:
178:   error:
179:     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "PETSc was not configured for PetscDeviceType %s", PetscDeviceTypes[dtype]);
180:   }
181:   PetscUseTypeMethod(device, configure);
182:   return 0;
183: }

185: /*@C
186:   PetscDeviceView - View a `PetscDevice`

188:   Collective on viewer

190:   Input Parameters:
191: + device - The `PetscDevice` to view
192: - viewer - The `PetscViewer` to view the device with (`NULL` for `PETSC_VIEWER_STDOUT_WORLD`)

194:   Level: beginner

196: .seealso: `PetscDevice`, `PetscDeviceCreate()`, `PetscDeviceConfigure()`,
197: `PetscDeviceDestroy()`, `PetscDeviceGetType()`, `PetscDeviceGetDeviceId()`
198: @*/
199: PetscErrorCode PetscDeviceView(PetscDevice device, PetscViewer viewer)
200: {
201:   auto      sub = viewer;
202:   PetscBool iascii;

205:   if (viewer) {
207:     PetscObjectTypeCompare(PetscObjectCast(viewer), PETSCVIEWERASCII, &iascii);
208:   } else {
209:     PetscViewerASCIIGetStdout(PETSC_COMM_WORLD, &viewer);
210:     iascii = PETSC_TRUE;
211:   }

213:   if (iascii) {
214:     auto        dtype = PETSC_DEVICE_HOST;
215:     MPI_Comm    comm;
216:     PetscMPIInt size;
217:     PetscInt    id = 0;

219:     PetscObjectGetComm(PetscObjectCast(viewer), &comm);
220:     MPI_Comm_size(comm, &size);

222:     PetscDeviceGetDeviceId(device, &id);
223:     PetscDeviceGetType(device, &dtype);
224:     PetscViewerGetSubViewer(viewer, PETSC_COMM_SELF, &sub);
225:     PetscViewerASCIIPrintf(sub, "PetscDevice Object: %d MPI %s\n", size, size == 1 ? "process" : "processes");
226:     PetscViewerASCIIPushTab(sub);
227:     PetscViewerASCIIPrintf(sub, "type: %s\n", PetscDeviceTypes[dtype]);
228:     PetscViewerASCIIPrintf(sub, "id: %" PetscInt_FMT "\n", id);
229:   }

231:   // see if impls has extra viewer stuff
232:   PetscTryTypeMethod(device, view, sub);

234:   if (iascii) {
235:     // undo the ASCII specific stuff
236:     PetscViewerASCIIPopTab(sub);
237:     PetscViewerRestoreSubViewer(viewer, PETSC_COMM_SELF, &sub);
238:     PetscViewerFlush(viewer);
239:   }
240:   return 0;
241: }

243: /*@C
244:   PetscDeviceGetType - Get the type of device

246:   Not Collective

248:   Input Parameter:
249: . device - The `PetscDevice`

251:   Output Parameter:
252: . type - The `PetscDeviceType`

254:   Level: beginner

256: .seealso: `PetscDevice`, `PetscDeviceType`, `PetscDeviceSetDefaultDeviceType()`,
257: `PetscDeviceCreate()`, `PetscDeviceConfigure()`, `PetscDeviceDestroy()`,
258: `PetscDeviceGetDeviceId()`, `PETSC_DEVICE_DEFAULT()`
259: @*/
260: PetscErrorCode PetscDeviceGetType(PetscDevice device, PetscDeviceType *type)
261: {
264:   *type = device->type;
265:   return 0;
266: }

268: /*@C
269:   PetscDeviceGetDeviceId - Get the device ID for a `PetscDevice`

271:   Not Collective

273:   Input Parameter:
274: . device - The `PetscDevice`

276:   Output Parameter:
277: . id - The id

279:   Notes:
280:   The returned ID may have been assigned by the underlying device backend. For example if the
281:   backend is CUDA then `id` is exactly the value returned by `cudaGetDevice()` at the time when
282:   this device was configured.

284:   Level: beginner

286: .seealso: `PetscDevice`, `PetscDeviceCreate()`, `PetscDeviceGetType()`
287: @*/
288: PetscErrorCode PetscDeviceGetDeviceId(PetscDevice device, PetscInt *id)
289: {
292:   *id = device->deviceId;
293:   return 0;
294: }

296: struct DefaultDeviceType : public Petsc::RegisterFinalizeable<DefaultDeviceType> {
297:   PetscDeviceType type = PETSC_DEVICE_HARDWARE_DEFAULT_TYPE;

299:   PETSC_NODISCARD PetscErrorCode finalize_() noexcept
300:   {
301:     type = PETSC_DEVICE_HARDWARE_DEFAULT_TYPE;
302:     return 0;
303:   }
304: };

306: static auto default_device_type = DefaultDeviceType();

308: /*@C
309:   PETSC_DEVICE_DEFAULT - Retrieve the current default `PetscDeviceType`

311:   Not Collective

313:   Notes:
314:   Unless selected by the user, the default device is selected in the following order\:
315:   `PETSC_DEVICE_HIP`, `PETSC_DEVICE_CUDA`, `PETSC_DEVICE_SYCL`, `PETSC_DEVICE_HOST`.

317:   Level: beginner

319: .seealso: `PetscDeviceType`, `PetscDeviceSetDefaultDeviceType()`, `PetscDeviceGetType()`
320: @*/
321: PetscDeviceType PETSC_DEVICE_DEFAULT(void)
322: {
323:   return default_device_type.type;
324: }

326: /*@C
327:   PetscDeviceSetDefaultDeviceType - Set the default device type for `PetscDevice`

329:   Not Collective

331:   Input Parameter:
332: . type - the new default device type

334:   Notes:
335:   This sets the `PetscDeviceType` returned by `PETSC_DEVICE_DEFAULT()`.

337:   Level: beginner

339: .seealso: `PetscDeviceType`, `PetscDeviceGetType`,
340: @*/
341: PetscErrorCode PetscDeviceSetDefaultDeviceType(PetscDeviceType type)
342: {
344:   if (default_device_type.type != type) {
345:     // no need to waster a PetscRegisterFinalize() slot if we don't change it
346:     default_device_type.type = type;
347:     default_device_type.register_finalize();
348:   }
349:   return 0;
350: }

352: static std::array<std::pair<PetscDevice, bool>, PETSC_DEVICE_MAX> defaultDevices = {};

354: /*
355:   Actual initialization function; any functions claiming to initialize PetscDevice or
356:   PetscDeviceContext will have to run through this one
357: */
358: static PetscErrorCode PetscDeviceInitializeDefaultDevice_Internal(PetscDeviceType type, PetscInt defaultDeviceId)
359: {
361:   if (PetscUnlikely(!PetscDeviceInitialized(type))) {
362:     auto &dev  = defaultDevices[type].first;
363:     auto &init = defaultDevices[type].second;

365:     PetscAssert(!dev, PETSC_COMM_SELF, PETSC_ERR_MEM, "Trying to overwrite existing default device of type %s", PetscDeviceTypes[type]);
366:     PetscDeviceCreate(type, defaultDeviceId, &dev);
367:     PetscDeviceConfigure(dev);
368:     init = true;
369:   }
370:   return 0;
371: }

373: /*@C
374:   PetscDeviceInitialize - Initialize `PetscDevice`

376:   Not Collective

378:   Input Parameter:
379: . type - The `PetscDeviceType` to initialize

381:   Notes:
382:   Eagerly initializes the corresponding `PetscDeviceType` if needed. If this is the case it may
383:   result in device synchronization.

385:   Level: beginner

387: .seealso: `PetscDevice`, `PetscDeviceInitType`, `PetscDeviceInitialized()`,
388: `PetscDeviceCreate()`, `PetscDeviceDestroy()`
389: @*/
390: PetscErrorCode PetscDeviceInitialize(PetscDeviceType type)
391: {
393:   PetscDeviceInitializeDefaultDevice_Internal(type, PETSC_DECIDE);
394:   return 0;
395: }

397: /*@C
398:   PetscDeviceInitialized - Determines whether `PetscDevice` is initialized for a particular
399:   `PetscDeviceType`

401:   Not Collective

403:   Input Parameter:
404: . type - The `PetscDeviceType` to check

406:   Notes:
407:   Returns `PETSC_TRUE` if `type` is initialized, `PETSC_FALSE` otherwise.

409:   If one has not configured PETSc for a particular `PetscDeviceType` then this routine will
410:   return `PETSC_FALSE` for that `PetscDeviceType`.

412:   Level: beginner

414: .seealso: `PetscDevice`, `PetscDeviceInitType`, `PetscDeviceInitialize()`,
415: `PetscDeviceCreate()`, `PetscDeviceDestroy()`
416: @*/
417: PetscBool PetscDeviceInitialized(PetscDeviceType type)
418: {
419:   return static_cast<PetscBool>(PetscDeviceConfiguredFor_Internal(type) && defaultDevices[type].second);
420: }

422: /* Get the default PetscDevice for a particular type and constructs them if lazily initialized. */
423: PetscErrorCode PetscDeviceGetDefaultForType_Internal(PetscDeviceType type, PetscDevice *device)
424: {
426:   PetscDeviceInitialize(type);
427:   *device = defaultDevices[type].first;
428:   return 0;
429: }

431: /*@C
432:   PetscDeviceGetAttribute - Query a particular attribute of a `PetscDevice`

434:   Not Collective

436:   Input Parameters:
437: + device - The `PetscDevice`
438: - attr   - The attribute

440:   Output Parameter:
441: . value - The value of the attribute

443:   Notes:
444:   Since different attributes are often different types `value` is a `void *` to accommodate
445:   them all. The underlying type of the attribute is therefore included in the name of the
446:   `PetscDeviceAttribute` responsible for querying it. For example,
447:   `PETSC_DEVICE_ATTR_SIZE_T_SHARED_MEM_PER_BLOCK` is of type `size_t`.

449:   Level: intermediate

451: .seealso: `PetscDeviceAtrtibute`, `PetscDeviceConfigure()`, `PetscDevice`
452: @*/
453: PetscErrorCode PetscDeviceGetAttribute(PetscDevice device, PetscDeviceAttribute attr, void *value)
454: {
458:   PetscUseTypeMethod(device, getattribute, attr, value);
459:   return 0;
460: }

462: static PetscErrorCode PetscDeviceInitializeTypeFromOptions_Private(MPI_Comm comm, PetscDeviceType type, PetscInt defaultDeviceId, PetscBool defaultView, PetscDeviceInitType *defaultInitType)
463: {
464:   if (!PetscDeviceConfiguredFor_Internal(type)) {
465:     PetscInfo(nullptr, "PetscDeviceType %s not available\n", PetscDeviceTypes[type]);
466:     defaultDevices[type].first = nullptr;
467:     return 0;
468:   }
469:   PetscInfo(nullptr, "PetscDeviceType %s available, initializing\n", PetscDeviceTypes[type]);
470:   /* ugly switch needed to pick the right global variable... could maybe do this as a union? */
471:   switch (type) {
472:     PETSC_DEVICE_CASE_IF_PETSC_DEFINED(HOST, initialize, comm, &defaultDeviceId, &defaultView, defaultInitType);
473:     PETSC_DEVICE_CASE_IF_PETSC_DEFINED(CUDA, initialize, comm, &defaultDeviceId, &defaultView, defaultInitType);
474:     PETSC_DEVICE_CASE_IF_PETSC_DEFINED(HIP, initialize, comm, &defaultDeviceId, &defaultView, defaultInitType);
475:     PETSC_DEVICE_CASE_IF_PETSC_DEFINED(SYCL, initialize, comm, &defaultDeviceId, &defaultView, defaultInitType);
476:   default:
477:     SETERRQ(comm, PETSC_ERR_PLIB, "PETSc was seemingly configured for PetscDeviceType %s but we've fallen through all cases in a switch", PetscDeviceTypes[type]);
478:   }
479:   PetscInfo(nullptr, "PetscDevice %s initialized, default device id %" PetscInt_FMT ", view %s, init type %s\n", PetscDeviceTypes[type], defaultDeviceId, PetscBools[defaultView], PetscDeviceInitTypes[Petsc::util::integral_value(*defaultInitType)]);
480:   /*
481:     defaultInitType, defaultView  and defaultDeviceId now represent what the individual TYPES
482:     have decided to initialize as
483:   */
484:   if ((*defaultInitType == PETSC_DEVICE_INIT_EAGER) || defaultView) {
485:     PetscInfo(nullptr, "Eagerly initializing %s PetscDevice\n", PetscDeviceTypes[type]);
486:     PetscDeviceInitializeDefaultDevice_Internal(type, defaultDeviceId);
487:     if (defaultView) PetscDeviceView(defaultDevices[type].first, nullptr);
488:   }
489:   return 0;
490: }

492: static PetscErrorCode PetscDeviceInitializeQueryOptions_Private(MPI_Comm comm, PetscDeviceType *deviceContextInitDevice, PetscDeviceInitType *defaultInitType, PetscInt *defaultDevice, PetscBool *defaultDeviceSet, PetscBool *defaultView)
493: {
494:   PetscInt initIdx       = PETSC_DEVICE_INIT_LAZY;
495:   auto     initDeviceIdx = static_cast<PetscInt>(*deviceContextInitDevice);
496:   auto     flg           = PETSC_FALSE;

498:   PetscOptionsHasName(nullptr, nullptr, "-log_view_gpu_time", &flg);
499:   if (flg) PetscLogGpuTime();

501:   PetscOptionsBegin(comm, nullptr, "PetscDevice Options", "Sys");
502:   PetscOptionsEList("-device_enable", "How (or whether) to initialize PetscDevices", "PetscDeviceInitialize()", PetscDeviceInitTypes, 3, PetscDeviceInitTypes[initIdx], &initIdx, nullptr);
503:   PetscOptionsEList("-default_device_type", "Set the PetscDeviceType returned by PETSC_DEVICE_DEFAULT()", "PetscDeviceSetDefaultDeviceType()", PetscDeviceTypes, PETSC_DEVICE_MAX, PetscDeviceTypes[initDeviceIdx], &initDeviceIdx, defaultDeviceSet);
504:   PetscOptionsRangeInt("-device_select", "Which device to use. Pass " PetscStringize(PETSC_DECIDE) " to have PETSc decide or (given they exist) [0-" PetscStringize(PETSC_DEVICE_MAX_DEVICES) ") for a specific device", "PetscDeviceCreate()", *defaultDevice, defaultDevice, nullptr, PETSC_DECIDE, PETSC_DEVICE_MAX_DEVICES);
505:   PetscOptionsBool("-device_view", "Display device information and assignments (forces eager initialization)", "PetscDeviceView()", *defaultView, defaultView, &flg);
506:   PetscOptionsEnd();

508:   if (initIdx == PETSC_DEVICE_INIT_NONE) {
509:     /* disabled all device initialization if devices are globally disabled */
511:     *defaultView  = PETSC_FALSE;
512:     initDeviceIdx = PETSC_DEVICE_HOST;
513:   } else {
514:     *defaultView = static_cast<PetscBool>(*defaultView && flg);
515:     if (*defaultView) initIdx = PETSC_DEVICE_INIT_EAGER;
516:   }
517:   *defaultInitType         = PetscDeviceInitTypeCast(initIdx);
518:   *deviceContextInitDevice = PetscDeviceTypeCast(initDeviceIdx);
519:   return 0;
520: }

522: /* called from PetscFinalize() do not call yourself! */
523: static PetscErrorCode PetscDeviceFinalize_Private()
524: {
525:   if (PetscDefined(USE_DEBUG)) {
526:     const auto PetscDeviceCheckAllDestroyedAfterFinalize = [] {
527:       for (auto &&device : defaultDevices) {
528:         const auto dev = device.first;

531:       }
532:       return 0;
533:     };
534:     /*
535:       you might be thinking, why on earth are you registered yet another finalizer in a
536:       function already called during PetscRegisterFinalizeAll()? If this seems stupid it's
537:       because it is.

539:       The crux of the problem is that the initializer (and therefore the ~finalizer~) of
540:       PetscDeviceContext is guaranteed to run after PetscDevice's. So if the global context had
541:       a default PetscDevice attached, that PetscDevice will have a reference count >0 and hence
542:       won't be destroyed yet. So we need to repeat the check that all devices have been
543:       destroyed again ~after~ the global context is destroyed. In summary:

545:       1. This finalizer runs and destroys all devices, except it may not because the global
546:          context may still hold a reference!
547:       2. The global context finalizer runs and does the final reference count decrement
548:          required, which actually destroys the held device.
549:       3. Our newly added finalizer runs and checks that all is well.
550:     */
551:     PetscRegisterFinalize(std::move(PetscDeviceCheckAllDestroyedAfterFinalize));
552:   }
553:   for (auto &&device : defaultDevices) {
554:     PetscDeviceDestroy(&device.first);
555:     device.second = false;
556:   }
557:   return 0;
558: }

560: /*
561:   Begins the init proceeedings for the entire PetscDevice stack. there are 3 stages of
562:   initialization types:

564:   1. defaultInitType - how does PetscDevice as a whole expect to initialize?
565:   2. subTypeDefaultInitType - how does each PetscDevice implementation expect to initialize?
566:      e.g. you may want to blanket disable PetscDevice init (and disable say Kokkos init), but
567:      have all CUDA devices still initialize.

569:   All told the following happens:

571:   0. defaultInitType -> LAZY
572:   1. Check for log_view/log_summary, if yes defaultInitType -> EAGER
573:   2. PetscDevice initializes each sub type with deviceDefaultInitType.
574:   2.1 Each enabled PetscDevice sub-type then does the above disable or view check in addition
575:       to checking for specific device init. if view or specific device init
576:       subTypeDefaultInitType -> EAGER. disabled once again overrides all.
577: */

579: PetscErrorCode PetscDeviceInitializeFromOptions_Internal(MPI_Comm comm)
580: {
581:   auto defaultView                    = PETSC_FALSE;
582:   auto initializeDeviceContextEagerly = PETSC_FALSE;
583:   auto defaultDeviceSet               = PETSC_FALSE;
584:   auto defaultDevice                  = PetscInt{PETSC_DECIDE};
585:   auto deviceContextInitDevice        = PETSC_DEVICE_DEFAULT();
586:   auto defaultInitType                = PETSC_DEVICE_INIT_LAZY;

588:   if (PetscDefined(USE_DEBUG)) {
589:     int result;

591:     MPI_Comm_compare(comm, PETSC_COMM_WORLD, &result);
592:     /* in order to accurately assign ranks to gpus we need to get the MPI_Comm_rank of the
593:      * global space */
594:     if (PetscUnlikely(result != MPI_IDENT)) {
595:       char name[MPI_MAX_OBJECT_NAME] = {};
596:       int  len; /* unused */

598:       MPI_Comm_get_name(comm, name, &len);
599:       SETERRQ(comm, PETSC_ERR_MPI, "Default devices being initialized on MPI_Comm '%s' not PETSC_COMM_WORLD", name);
600:     }
601:   }
602:   comm = PETSC_COMM_WORLD; /* from this point on we assume we're on PETSC_COMM_WORLD */
603:   PetscRegisterFinalize(PetscDeviceFinalize_Private);

605:   PetscDeviceInitializeQueryOptions_Private(comm, &deviceContextInitDevice, &defaultInitType, &defaultDevice, &defaultDeviceSet, &defaultView);

607:   // the precise values don't matter here, so long as they are sequential
608:   static_assert(Petsc::util::integral_value(PETSC_DEVICE_HOST) == 0, "");
609:   static_assert(Petsc::util::integral_value(PETSC_DEVICE_CUDA) == 1, "");
610:   static_assert(Petsc::util::integral_value(PETSC_DEVICE_HIP) == 2, "");
611:   static_assert(Petsc::util::integral_value(PETSC_DEVICE_SYCL) == 3, "");
612:   static_assert(Petsc::util::integral_value(PETSC_DEVICE_MAX) == 4, "");
613:   for (int i = PETSC_DEVICE_HOST; i < PETSC_DEVICE_MAX; ++i) {
614:     const auto deviceType = PetscDeviceTypeCast(i);
615:     auto       initType   = defaultInitType;

617:     PetscDeviceInitializeTypeFromOptions_Private(comm, deviceType, defaultDevice, defaultView, &initType);
618:     if (PetscDeviceConfiguredFor_Internal(deviceType)) {
619:       if (initType == PETSC_DEVICE_INIT_EAGER) {
620:         initializeDeviceContextEagerly = PETSC_TRUE;
621:         // only update the default device if the user hasn't set it previously
622:         if (!defaultDeviceSet) {
623:           deviceContextInitDevice = deviceType;
624:           PetscInfo(nullptr, "PetscDevice %s set as default device type due to eager initialization\n", PetscDeviceTypes[deviceType]);
625:         }
626:       } else if (initType == PETSC_DEVICE_INIT_NONE) {
628:       }
629:     }
630:   }

632:   PetscDeviceSetDefaultDeviceType(deviceContextInitDevice);
633:   PetscDeviceContextSetRootDeviceType_Internal(PETSC_DEVICE_DEFAULT());
634:   /* ----------------------------------------------------------------------------------- */
635:   /*                       PetscDevice is now fully initialized                          */
636:   /* ----------------------------------------------------------------------------------- */
637:   {
638:     /*
639:       query the options db to get the root settings from the user (if any).

641:       This section is a bit of a hack. We have to reach across to dcontext.cxx to all but call
642:       PetscDeviceContextSetFromOptions() before we even have one, then set a few static
643:       variables in that file with the results.
644:     */
645:     auto dtype = std::make_pair(PETSC_DEVICE_DEFAULT(), PETSC_FALSE);
646:     auto stype = std::make_pair(PETSC_DEVICE_CONTEXT_DEFAULT_STREAM_TYPE, PETSC_FALSE);

648:     PetscOptionsBegin(comm, "root_", "Root PetscDeviceContext Options", "Sys");
649:     PetscDeviceContextQueryOptions_Internal(PetscOptionsObject, dtype, stype);
650:     PetscOptionsEnd();

652:     if (dtype.second) PetscDeviceContextSetRootDeviceType_Internal(dtype.first);
653:     if (stype.second) PetscDeviceContextSetRootStreamType_Internal(stype.first);
654:   }

656:   if (initializeDeviceContextEagerly) {
657:     PetscDeviceContext dctx;

659:     PetscInfo(nullptr, "Eagerly initializing PetscDeviceContext with %s device\n", PetscDeviceTypes[deviceContextInitDevice]);
660:     /* instantiates the device context */
661:     PetscDeviceContextGetCurrentContext(&dctx);
662:     PetscDeviceContextSetUp(dctx);
663:   }
664:   return 0;
665: }