Actual source code: sycldevice.sycl.cxx
1: #include "sycldevice.hpp"
2: #include <limits> // for std::numeric_limits
3: #include <csetjmp> // for MPI sycl device awareness
4: #include <csignal> // SIGSEGV
5: #include <vector>
6: #include <CL/sycl.hpp>
8: namespace Petsc
9: {
11: namespace device
12: {
14: namespace sycl
15: {
17: // definition for static
18: std::array<Device::DeviceInternal *, PETSC_DEVICE_MAX_DEVICES> Device::devices_array_ = {};
19: Device::DeviceInternal **Device::devices_ = &Device::devices_array_[1];
20: int Device::defaultDevice_ = PETSC_SYCL_DEVICE_NONE;
21: bool Device::initialized_ = false;
23: static std::jmp_buf MPISyclAwareJumpBuffer;
24: static bool MPISyclAwareJumpBufferSet;
26: // internal "impls" class for SyclDevice. Each instance represents a single sycl device
27: class PETSC_NODISCARD Device::DeviceInternal {
28: const int id_; // -1 for the host device; 0 and up for gpu devices
29: bool devInitialized_;
30: const ::sycl::device syclDevice_;
32: public:
33: // default constructor
34: DeviceInternal(int id) noexcept : id_(id), devInitialized_(false), syclDevice_(chooseSYCLDevice_(id)) { }
35: int id() const { return id_; }
36: bool initialized() const { return devInitialized_; }
38: PETSC_NODISCARD PetscErrorCode initialize() noexcept
39: {
40: if (initialized()) return 0;
41: if (syclDevice_.is_gpu() && use_gpu_aware_mpi) {
42: if (!isMPISyclAware_()) {
43: (*PetscErrorPrintf)("PETSc is configured with sycl support, but your MPI is not aware of sycl GPU devices. For better performance, please use a sycl GPU-aware MPI.\n");
44: (*PetscErrorPrintf)("If you do not care, add option -use_gpu_aware_mpi 0. To not see the message again, add the option to your .petscrc, OR add it to the env var PETSC_OPTIONS.\n");
45: PETSCABORT(PETSC_COMM_SELF, PETSC_ERR_LIB);
46: }
47: }
48: devInitialized_ = true;
49: return 0;
50: }
52: PETSC_NODISCARD PetscErrorCode view(PetscViewer viewer) const noexcept
53: {
54: MPI_Comm comm;
55: PetscMPIInt rank;
56: PetscBool iascii;
59: PetscObjectTypeCompare(reinterpret_cast<PetscObject>(viewer), PETSCVIEWERASCII, &iascii);
60: PetscObjectGetComm(reinterpret_cast<PetscObject>(viewer), &comm);
61: if (iascii) {
62: PetscViewer sviewer;
64: MPI_Comm_rank(comm, &rank);
65: PetscViewerGetSubViewer(viewer, PETSC_COMM_SELF, &sviewer);
66: PetscViewerASCIIPrintf(sviewer, "[%d] device: %s\n", rank, syclDevice_.get_info<::sycl::info::device::name>().c_str());
67: PetscViewerASCIIPushTab(sviewer);
68: PetscViewerASCIIPrintf(sviewer, "-> Device vendor: %s\n", syclDevice_.get_info<::sycl::info::device::vendor>().c_str());
69: PetscViewerASCIIPopTab(sviewer);
70: PetscViewerFlush(sviewer);
71: PetscViewerRestoreSubViewer(viewer, PETSC_COMM_SELF, &sviewer);
72: PetscViewerFlush(viewer);
73: }
74: return 0;
75: }
77: PETSC_NODISCARD PetscErrorCode getattribute(PetscDeviceAttribute attr, void *value) const noexcept
78: {
80: switch (attr) {
81: case PETSC_DEVICE_ATTR_SIZE_T_SHARED_MEM_PER_BLOCK:
82: *static_cast<std::size_t *>(value) = syclDevice_.get_info<::sycl::info::device::local_mem_size>();
83: case PETSC_DEVICE_ATTR_MAX:
84: break;
85: }
86: return 0;
87: }
89: private:
90: static ::sycl::device chooseSYCLDevice_(int id)
91: {
92: if (id == PETSC_SYCL_DEVICE_HOST) {
93: return ::sycl::device(::sycl::host_selector());
94: } else {
95: return ::sycl::device::get_devices(::sycl::info::device_type::gpu)[id];
96: }
97: }
99: // Is the underlying MPI aware of sycl (GPU) devices?
100: bool isMPISyclAware_() noexcept
101: {
102: const int bufSize = 2;
103: const int hbuf[bufSize] = {1, 0};
104: int *dbuf = nullptr;
105: bool awareness = false;
106: const auto SyclSignalHandler = [](int signal, void *ptr) -> PetscErrorCode {
107: if ((signal == SIGSEGV) && MPISyclAwareJumpBufferSet) std::longjmp(MPISyclAwareJumpBuffer, 1);
108: return PetscSignalHandlerDefault(signal, ptr);
109: };
111: auto Q = ::sycl::queue(syclDevice_);
112: dbuf = ::sycl::malloc_device<int>(bufSize, Q);
113: Q.memcpy(dbuf, hbuf, sizeof(int) * bufSize).wait();
114: PETSC_COMM_SELF, PetscPushSignalHandler(SyclSignalHandler, nullptr);
115: MPISyclAwareJumpBufferSet = true;
116: if (setjmp(MPISyclAwareJumpBuffer)) {
117: // if a segv was triggered in the MPI_Allreduce below, it is very likely due to MPI not being GPU-aware
118: awareness = false;
119: PetscStackPop;
120: } else if (!MPI_Allreduce(dbuf, dbuf + 1, 1, MPI_INT, MPI_SUM, PETSC_COMM_SELF)) awareness = true;
121: MPISyclAwareJumpBufferSet = false;
122: PETSC_COMM_SELF, PetscPopSignalHandler();
123: ::sycl::free(dbuf, Q);
124: return awareness;
125: }
126: };
128: PetscErrorCode Device::initialize(MPI_Comm comm, PetscInt *defaultDeviceId, PetscBool *defaultView, PetscDeviceInitType *defaultInitType) noexcept
129: {
130: auto id = *defaultDeviceId;
131: auto initType = *defaultInitType;
132: auto view = *defaultView, flg = PETSC_FALSE;
133: PetscInt ngpus;
135: if (initialized_) return 0;
136: initialized_ = true;
137: PetscRegisterFinalize(finalize_);
138: PetscOptionsBegin(comm, nullptr, "PetscDevice sycl Options", "Sys");
139: base_type::PetscOptionDeviceInitialize(PetscOptionsObject, &initType, nullptr);
140: base_type::PetscOptionDeviceSelect(PetscOptionsObject, "Which sycl device to use? Pass -2 for host, PETSC_DECIDE (" PetscStringize(PETSC_DECIDE) ") to let PETSc decide, 0 and up for GPUs", "PetscDeviceCreate()", id, &id, nullptr, -2, std::numeric_limits<decltype(ngpus)>::max());
141: static_assert(PETSC_DECIDE - 1 == -2, "");
142: base_type::PetscOptionDeviceView(PetscOptionsObject, &view, &flg);
143: PetscOptionsEnd();
145: // post-process the options and lay the groundwork for initialization if needs be
146: std::vector<::sycl::device> gpu_devices = ::sycl::device::get_devices(::sycl::info::device_type::gpu);
147: ngpus = static_cast<PetscInt>(gpu_devices.size());
151: if (initType == PETSC_DEVICE_INIT_NONE) id = PETSC_SYCL_DEVICE_NONE; /* user wants to disable all sycl devices */
152: else {
153: PetscDeviceCheckDeviceCount_Internal(ngpus);
154: if (id == PETSC_DECIDE) { /* petsc will choose a GPU device if any, otherwise a CPU device */
155: if (ngpus) {
156: PetscMPIInt rank;
157: MPI_Comm_rank(comm, &rank);
158: id = rank % ngpus;
159: } else id = PETSC_SYCL_DEVICE_HOST;
160: }
161: if (view) initType = PETSC_DEVICE_INIT_EAGER;
162: }
164: if (id == -2) id = PETSC_SYCL_DEVICE_HOST; // user passed in '-device_select_sycl -2'. We transform it into canonical form
166: defaultDevice_ = static_cast<decltype(defaultDevice_)>(id);
168: // record the results of the initialization
169: *defaultDeviceId = id;
170: *defaultView = view;
171: *defaultInitType = initType;
172: return 0;
173: }
175: PetscErrorCode Device::finalize_() noexcept
176: {
177: if (!initialized_) return 0;
178: for (auto &&devPtr : devices_array_) delete devPtr;
179: defaultDevice_ = PETSC_SYCL_DEVICE_NONE; // disabled by default
180: initialized_ = false;
181: return 0;
182: }
184: PetscErrorCode Device::init_device_id_(PetscInt *inid) const noexcept
185: {
186: const auto id = *inid == PETSC_DECIDE ? defaultDevice_ : *inid;
190: devices_array_.size(), id);
191: if (!devices_[id]) devices_[id] = new DeviceInternal(id);
193: devices_[id]->initialize();
194: *inid = id;
195: return 0;
196: }
198: PetscErrorCode Device::view_device_(PetscDevice device, PetscViewer viewer) noexcept
199: {
200: devices_[device->deviceId]->view(viewer);
201: return 0;
202: }
204: PetscErrorCode Device::get_attribute_(PetscInt id, PetscDeviceAttribute attr, void *value) noexcept
205: {
206: devices_[id]->getattribute(attr, value);
207: return 0;
208: }
210: } // namespace sycl
212: } // namespace device
214: } // namespace Petsc