Actual source code: sycldevice.sycl.cxx

  1: #include "sycldevice.hpp"
  2: #include <limits>  // for std::numeric_limits
  3: #include <csetjmp> // for MPI sycl device awareness
  4: #include <csignal> // SIGSEGV
  5: #include <vector>
  6: #include <CL/sycl.hpp>

  8: namespace Petsc
  9: {

 11: namespace device
 12: {

 14: namespace sycl
 15: {

 17: // definition for static
 18: std::array<Device::DeviceInternal *, PETSC_DEVICE_MAX_DEVICES> Device::devices_array_ = {};
 19: Device::DeviceInternal                                       **Device::devices_       = &Device::devices_array_[1];
 20: int                                                            Device::defaultDevice_ = PETSC_SYCL_DEVICE_NONE;
 21: bool                                                           Device::initialized_   = false;

 23: static std::jmp_buf MPISyclAwareJumpBuffer;
 24: static bool         MPISyclAwareJumpBufferSet;

 26: // internal "impls" class for SyclDevice. Each instance represents a single sycl device
 27: class PETSC_NODISCARD Device::DeviceInternal {
 28:   const int            id_; // -1 for the host device; 0 and up for gpu devices
 29:   bool                 devInitialized_;
 30:   const ::sycl::device syclDevice_;

 32: public:
 33:   // default constructor
 34:   DeviceInternal(int id) noexcept : id_(id), devInitialized_(false), syclDevice_(chooseSYCLDevice_(id)) { }
 35:   int  id() const { return id_; }
 36:   bool initialized() const { return devInitialized_; }

 38:   PETSC_NODISCARD PetscErrorCode initialize() noexcept
 39:   {
 40:     if (initialized()) return 0;
 41:     if (syclDevice_.is_gpu() && use_gpu_aware_mpi) {
 42:       if (!isMPISyclAware_()) {
 43:         (*PetscErrorPrintf)("PETSc is configured with sycl support, but your MPI is not aware of sycl GPU devices. For better performance, please use a sycl GPU-aware MPI.\n");
 44:         (*PetscErrorPrintf)("If you do not care, add option -use_gpu_aware_mpi 0. To not see the message again, add the option to your .petscrc, OR add it to the env var PETSC_OPTIONS.\n");
 45:         PETSCABORT(PETSC_COMM_SELF, PETSC_ERR_LIB);
 46:       }
 47:     }
 48:     devInitialized_ = true;
 49:     return 0;
 50:   }

 52:   PETSC_NODISCARD PetscErrorCode view(PetscViewer viewer) const noexcept
 53:   {
 54:     MPI_Comm    comm;
 55:     PetscMPIInt rank;
 56:     PetscBool   iascii;

 59:     PetscObjectTypeCompare(reinterpret_cast<PetscObject>(viewer), PETSCVIEWERASCII, &iascii);
 60:     PetscObjectGetComm(reinterpret_cast<PetscObject>(viewer), &comm);
 61:     if (iascii) {
 62:       PetscViewer sviewer;

 64:       MPI_Comm_rank(comm, &rank);
 65:       PetscViewerGetSubViewer(viewer, PETSC_COMM_SELF, &sviewer);
 66:       PetscViewerASCIIPrintf(sviewer, "[%d] device: %s\n", rank, syclDevice_.get_info<::sycl::info::device::name>().c_str());
 67:       PetscViewerASCIIPushTab(sviewer);
 68:       PetscViewerASCIIPrintf(sviewer, "-> Device vendor: %s\n", syclDevice_.get_info<::sycl::info::device::vendor>().c_str());
 69:       PetscViewerASCIIPopTab(sviewer);
 70:       PetscViewerFlush(sviewer);
 71:       PetscViewerRestoreSubViewer(viewer, PETSC_COMM_SELF, &sviewer);
 72:       PetscViewerFlush(viewer);
 73:     }
 74:     return 0;
 75:   }

 77:   PETSC_NODISCARD PetscErrorCode getattribute(PetscDeviceAttribute attr, void *value) const noexcept
 78:   {
 80:     switch (attr) {
 81:     case PETSC_DEVICE_ATTR_SIZE_T_SHARED_MEM_PER_BLOCK:
 82:       *static_cast<std::size_t *>(value) = syclDevice_.get_info<::sycl::info::device::local_mem_size>();
 83:     case PETSC_DEVICE_ATTR_MAX:
 84:       break;
 85:     }
 86:     return 0;
 87:   }

 89: private:
 90:   static ::sycl::device chooseSYCLDevice_(int id)
 91:   {
 92:     if (id == PETSC_SYCL_DEVICE_HOST) {
 93:       return ::sycl::device(::sycl::host_selector());
 94:     } else {
 95:       return ::sycl::device::get_devices(::sycl::info::device_type::gpu)[id];
 96:     }
 97:   }

 99:   // Is the underlying MPI aware of sycl (GPU) devices?
100:   bool isMPISyclAware_() noexcept
101:   {
102:     const int  bufSize           = 2;
103:     const int  hbuf[bufSize]     = {1, 0};
104:     int       *dbuf              = nullptr;
105:     bool       awareness         = false;
106:     const auto SyclSignalHandler = [](int signal, void *ptr) -> PetscErrorCode {
107:       if ((signal == SIGSEGV) && MPISyclAwareJumpBufferSet) std::longjmp(MPISyclAwareJumpBuffer, 1);
108:       return PetscSignalHandlerDefault(signal, ptr);
109:     };

111:     auto Q = ::sycl::queue(syclDevice_);
112:     dbuf   = ::sycl::malloc_device<int>(bufSize, Q);
113:     Q.memcpy(dbuf, hbuf, sizeof(int) * bufSize).wait();
114:     PETSC_COMM_SELF, PetscPushSignalHandler(SyclSignalHandler, nullptr);
115:     MPISyclAwareJumpBufferSet = true;
116:     if (setjmp(MPISyclAwareJumpBuffer)) {
117:       // if a segv was triggered in the MPI_Allreduce below, it is very likely due to MPI not being GPU-aware
118:       awareness = false;
119:       PetscStackPop;
120:     } else if (!MPI_Allreduce(dbuf, dbuf + 1, 1, MPI_INT, MPI_SUM, PETSC_COMM_SELF)) awareness = true;
121:     MPISyclAwareJumpBufferSet = false;
122:     PETSC_COMM_SELF, PetscPopSignalHandler();
123:     ::sycl::free(dbuf, Q);
124:     return awareness;
125:   }
126: };

128: PetscErrorCode Device::initialize(MPI_Comm comm, PetscInt *defaultDeviceId, PetscBool *defaultView, PetscDeviceInitType *defaultInitType) noexcept
129: {
130:   auto     id       = *defaultDeviceId;
131:   auto     initType = *defaultInitType;
132:   auto     view = *defaultView, flg = PETSC_FALSE;
133:   PetscInt ngpus;

135:   if (initialized_) return 0;
136:   initialized_ = true;
137:   PetscRegisterFinalize(finalize_);
138:   PetscOptionsBegin(comm, nullptr, "PetscDevice sycl Options", "Sys");
139:   base_type::PetscOptionDeviceInitialize(PetscOptionsObject, &initType, nullptr);
140:   base_type::PetscOptionDeviceSelect(PetscOptionsObject, "Which sycl device to use? Pass -2 for host, PETSC_DECIDE (" PetscStringize(PETSC_DECIDE) ") to let PETSc decide, 0 and up for GPUs", "PetscDeviceCreate()", id, &id, nullptr, -2, std::numeric_limits<decltype(ngpus)>::max());
141:   static_assert(PETSC_DECIDE - 1 == -2, "");
142:   base_type::PetscOptionDeviceView(PetscOptionsObject, &view, &flg);
143:   PetscOptionsEnd();

145:   // post-process the options and lay the groundwork for initialization if needs be
146:   std::vector<::sycl::device> gpu_devices = ::sycl::device::get_devices(::sycl::info::device_type::gpu);
147:   ngpus                                   = static_cast<PetscInt>(gpu_devices.size());

151:   if (initType == PETSC_DEVICE_INIT_NONE) id = PETSC_SYCL_DEVICE_NONE; /* user wants to disable all sycl devices */
152:   else {
153:     PetscDeviceCheckDeviceCount_Internal(ngpus);
154:     if (id == PETSC_DECIDE) { /* petsc will choose a GPU device if any, otherwise a CPU device */
155:       if (ngpus) {
156:         PetscMPIInt rank;
157:         MPI_Comm_rank(comm, &rank);
158:         id = rank % ngpus;
159:       } else id = PETSC_SYCL_DEVICE_HOST;
160:     }
161:     if (view) initType = PETSC_DEVICE_INIT_EAGER;
162:   }

164:   if (id == -2) id = PETSC_SYCL_DEVICE_HOST; // user passed in '-device_select_sycl -2'. We transform it into canonical form

166:   defaultDevice_ = static_cast<decltype(defaultDevice_)>(id);
168:   // record the results of the initialization
169:   *defaultDeviceId = id;
170:   *defaultView     = view;
171:   *defaultInitType = initType;
172:   return 0;
173: }

175: PetscErrorCode Device::finalize_() noexcept
176: {
177:   if (!initialized_) return 0;
178:   for (auto &&devPtr : devices_array_) delete devPtr;
179:   defaultDevice_ = PETSC_SYCL_DEVICE_NONE; // disabled by default
180:   initialized_   = false;
181:   return 0;
182: }

184: PetscErrorCode Device::init_device_id_(PetscInt *inid) const noexcept
185: {
186:   const auto id = *inid == PETSC_DECIDE ? defaultDevice_ : *inid;

190:              devices_array_.size(), id);
191:   if (!devices_[id]) devices_[id] = new DeviceInternal(id);
193:   devices_[id]->initialize();
194:   *inid = id;
195:   return 0;
196: }

198: PetscErrorCode Device::view_device_(PetscDevice device, PetscViewer viewer) noexcept
199: {
200:   devices_[device->deviceId]->view(viewer);
201:   return 0;
202: }

204: PetscErrorCode Device::get_attribute_(PetscInt id, PetscDeviceAttribute attr, void *value) noexcept
205: {
206:   devices_[id]->getattribute(attr, value);
207:   return 0;
208: }

210: } // namespace sycl

212: } // namespace device

214: } // namespace Petsc