XRootD
Loading...
Searching...
No Matches
XrdOssMio.cc
Go to the documentation of this file.
1/******************************************************************************/
2/* */
3/* X r d O s s M i o . c c */
4/* */
5/* (c) 2005 by the Board of Trustees of the Leland Stanford, Jr., University */
6/* All Rights Reserved */
7/* Produced by Andrew Hanushevsky for Stanford University under contract */
8/* DE-AC02-76-SFO0515 with the Department of Energy */
9/* */
10/* This file is part of the XRootD software suite. */
11/* */
12/* XRootD is free software: you can redistribute it and/or modify it under */
13/* the terms of the GNU Lesser General Public License as published by the */
14/* Free Software Foundation, either version 3 of the License, or (at your */
15/* option) any later version. */
16/* */
17/* XRootD is distributed in the hope that it will be useful, but WITHOUT */
18/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */
19/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */
20/* License for more details. */
21/* */
22/* You should have received a copy of the GNU Lesser General Public License */
23/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */
24/* COPYING (GPL license). If not, see <http://www.gnu.org/licenses/>. */
25/* */
26/* The copyright holder's institutional names and contributor's names may not */
27/* be used to endorse or promote products derived from this software without */
28/* specific prior written permission of the institution or contributor. */
29/******************************************************************************/
30
31#include <unistd.h>
32#include <cstdio>
33#include <sys/param.h>
34#include <sys/types.h>
35#include <sys/stat.h>
36
37#if defined(_POSIX_MAPPED_FILES)
38#include <sys/mman.h>
39#endif
40
42#include "XrdOss/XrdOssMio.hh"
44#include "XrdOss/XrdOssTrace.hh"
45#include "XrdOuc/XrdOucUtils.hh"
46
47/******************************************************************************/
48/* S t a t i c V a r i a b l e s */
49/******************************************************************************/
50
51XrdOucHash<XrdOssMioFile> XrdOssMio::MM_Hash;
52
53XrdSysMutex XrdOssMio::MM_Mutex;
54
55XrdOssMioFile *XrdOssMio::MM_Perm = 0;
56XrdOssMioFile *XrdOssMio::MM_Idle = 0;
57XrdOssMioFile *XrdOssMio::MM_IdleLast = 0;
58
59char XrdOssMio::MM_on = 1;
60char XrdOssMio::MM_chk = 0;
61char XrdOssMio::MM_okmlock = 1;
62char XrdOssMio::MM_preld = 0;
63long long XrdOssMio::MM_pagsz = (long long)sysconf(_SC_PAGESIZE);
64#ifdef __APPLE__
65long long XrdOssMio::MM_pages = 1024*1024*1024;
66#else
67long long XrdOssMio::MM_pages = (long long)sysconf(_SC_PHYS_PAGES);
68#endif
69long long XrdOssMio::MM_max = MM_pagsz*MM_pages/2;
70long long XrdOssMio::MM_inuse = 0;
71
73
75
76/******************************************************************************/
77/* D i s p l a y */
78/******************************************************************************/
79
81{
82 char buff[1080];
83 snprintf(buff, sizeof(buff), " oss.memfile %s%s%s max %lld",
84 (MM_on ? "" : "off "),
85 (MM_preld ? "preload" : ""),
86 (MM_chk ? "check xattr" : ""), MM_max);
87 Eroute.Say(buff);
88}
89
90/******************************************************************************/
91/* M a p */
92/******************************************************************************/
93
94XrdOssMioFile *XrdOssMio::Map(char *path, int fd, int opts)
95{
96#if defined(_POSIX_MAPPED_FILES)
97 EPNAME("MioMap");
98 XrdSysMutexHelper mapMutex;
99 struct stat statb;
100 XrdOssMioFile *mp;
101 void *thefile;
102 char hashname[64];
103
104// Get the size of the file
105//
106 if (fstat(fd, &statb))
107 {OssEroute.Emsg("Mio", errno, "fstat file", path);
108 return 0;
109 }
110
111// Develop hash name for this file
112//
113 int st_devSZ = sizeof(statb.st_dev);
114 XrdOucUtils::bin2hex((char *)&statb.st_dev, st_devSZ,
115 hashname, sizeof(hashname), false);
116 st_devSZ <<= 1;
117 XrdOucUtils::bin2hex((char *)&statb.st_ino, int(sizeof(statb.st_ino)),
118 hashname+st_devSZ, sizeof(hashname) - st_devSZ, false);
119
120// Because of potntial race conditions, we must serialize execution
121//
122 mapMutex.Lock(&MM_Mutex);
123
124// Check if we already have this mapping
125//
126 if ((mp = MM_Hash.Find(hashname)))
127 {DEBUG("Reusing mmap; usecnt=" <<mp->inUse <<" path=" <<path);
128 if (!(mp->Status & OSSMIO_MPRM) && !mp->inUse) Reclaim(mp);
129 mp->inUse++;
130 return mp;
131 }
132
133// Check if memory will be over committed
134//
135 if (MM_inuse + statb.st_size > MM_max)
136 {if (!Reclaim(statb.st_size))
137 {OssEroute.Emsg("Mio", "Unable to reclaim enough storage to mmap",path);
138 return 0;
139 }
140 }
141 MM_inuse += statb.st_size;
142
143// Memory map the file
144//
145 if ((thefile = mmap(0,statb.st_size,PROT_READ,MAP_PRIVATE,fd,0))==MAP_FAILED)
146 {OssEroute.Emsg("Mio", errno, "mmap file", path);
147 return 0;
148 } else {DEBUG("mmap " <<statb.st_size <<" bytes for " <<path);}
149
150// Lock the file, if need be. Turn off locking if we don't have privs
151//
152 if (MM_okmlock && (opts & OSSMIO_MLOK))
153 {if (mlock((char *)thefile, statb.st_size))
154 { if (errno == ENOSYS || errno == ENOTSUP)
155 {OssEroute.Emsg("Mio","mlock() not supported; feature disabled.");
156 MM_okmlock = 0;
157 }
158 else if (errno == EPERM)
159 {OssEroute.Emsg("Mio","Not privileged for mlock(); feature disabled.");
160 MM_okmlock = 0;
161 }
162 else OssEroute.Emsg("Mio", errno, "mlock file", path);
163 } else {DEBUG("Locked " <<statb.st_size <<" bytes for " <<path);}
164 }
165
166// get a new file object
167//
168 if (!(mp = new XrdOssMioFile(hashname)))
169 {OssEroute.Emsg("Mio", "Unable to allocate mmap file object for", path);
170 munmap((char *)thefile, statb.st_size);
171 return 0;
172 }
173
174// Complete the object here
175//
176 mp->Base = thefile;
177 mp->Size = statb.st_size;
178 mp->Dev = statb.st_dev;
179 mp->Ino = statb.st_ino;
180 mp->Status = opts;
181
182// Add the mapping to our hash table
183//
184 if (MM_Hash.Add(hashname, mp))
185 {OssEroute.Emsg("Mio", "Hash add failed for", path);
186 munmap((char *)thefile, statb.st_size);
187 delete mp;
188 return 0;
189 }
190
191// If this is a permanent file, place it on the permanent queue
192//
193 if (opts & OSSMIO_MPRM)
194 {mp->Next = MM_Perm; MM_Perm = mp;
195 DEBUG("Placed file on permanent queue " <<path);
196 }
197
198// If this file is to be preloaded, start it now
199//
200 if (MM_preld && mp->inUse == 1)
201 {pthread_t tid;
202 int retc;
203 mp->inUse++;
204 if ((retc = XrdSysThread::Run(&tid, preLoad, (void *)mp)) < 0)
205 {OssEroute.Emsg("Mio", retc, "creating mmap preload thread");
206 mp->inUse--;
207 }
208 else DEBUG("started mmap preload thread; tid=" <<(unsigned long)tid);
209 }
210
211// All done
212//
213 return mp;
214#else
215 return 0;
216#endif
217}
218
219/******************************************************************************/
220/* p r e L o a d */
221/******************************************************************************/
222
223void *XrdOssMio::preLoad(void *arg)
224{
225 XrdOssMioFile *mp = (XrdOssMioFile *)arg;
226 char *Base = (char *)(mp->Base);
227 char *Bend = Base + mp->Size;
228 long long MY_pagsz = MM_pagsz;
229
230// Reference each page until we are done. This is somewhat obtuse but we
231// are trying to keep the compiler from optimizing out the code.
232//
233 while(Base < Bend) Base += (*Base ? MY_pagsz : MM_pagsz);
234
235// All done
236//
237 Recycle(mp);
238 return (void *)0;
239}
240
241/******************************************************************************/
242/* R e c l a i m */
243/******************************************************************************/
244
245// Reclaim() can only be called if the caller has the MM_Mutex lock!
246//
247int XrdOssMio::Reclaim(off_t amount)
248{
249 EPNAME("MioReclaim");
250 XrdOssMioFile *mp;
251 DEBUG("Trying to reclaim " <<amount <<" bytes.");
252
253// Try to reclaim memory
254//
255 while((mp = MM_Idle) && amount > 0)
256 {MM_Idle = mp->Next;
257 MM_inuse -= mp->Size;
258 amount -= mp->Size;
259 MM_Hash.Del(mp->HashName); // This will delete the object
260 }
261
262// Indicate whether we cleared enough
263//
264 return amount <= 0;
265}
266
267/******************************************************************************/
268
269int XrdOssMio::Reclaim(XrdOssMioFile *mp)
270{
271 EPNAME("MioReclaim");
272 XrdOssMioFile *pmp = 0, *cmp = MM_Idle;
273
274// Try to find the mapping
275//
276 while(cmp && mp != cmp) {pmp = cmp; cmp = cmp->Next;}
277
278// Remove mapping from the idle list
279//
280 if (cmp)
281 {if (pmp) pmp->Next = mp->Next;
282 else MM_Idle = mp->Next;
283 if (MM_IdleLast == cmp) MM_IdleLast = pmp;
284 }
285 else {DEBUG("Cannot find mapping for " <<mp->Dev <<':' <<mp->Ino);}
286
287 return (cmp != 0);
288}
289
290/******************************************************************************/
291/* R e c y c l e */
292/******************************************************************************/
293
295{
296 XrdSysMutexHelper mmMutex(&MM_Mutex);
297
298// Decrement the use count
299//
300 mp->inUse--;
301 if (mp->inUse < 0)
302 {OssEroute.Emsg("Mio", "MM usecount underflow for ", mp->HashName);
303 mp->inUse = 0;
304 } else if (mp->inUse > 0) return;
305
306// If this is not a kept mapping, put it on the reclaim list
307//
308 if (!(mp->Status & OSSMIO_MPRM))
309 {if (MM_IdleLast) MM_IdleLast->Next = mp;
310 else MM_Idle = mp;
311 MM_IdleLast = mp;
312 mp->Next = 0;
313 }
314}
315
316/******************************************************************************/
317/* S e t */
318/******************************************************************************/
319
320void XrdOssMio::Set(int V_on, int V_preld, int V_check)
321{
322 if (V_on >= 0) MM_on = (char)V_on;
323 if (V_preld >= 0) MM_preld = (char)V_preld;
324 if (V_check >= 0) MM_chk = (char)V_check;
325}
326
327void XrdOssMio::Set(long long V_max)
328{
329 if (V_max > 0) MM_max = V_max;
330 else if (V_max < 0) MM_max = MM_pagsz*MM_pages*(-V_max)/100;
331}
332
333/******************************************************************************/
334/* X r d O s s d M i o F i l e D e s t r u c t o r */
335/******************************************************************************/
336
338{
339#if defined(_POSIX_MAPPED_FILES)
340 munmap((char *)Base, Size);
341#endif
342}
#define DEBUG(x)
#define EPNAME(x)
XrdSysError OssEroute
XrdSysTrace OssTrace
XrdSysError OssEroute
#define OSSMIO_MLOK
Definition XrdOssMio.hh:40
#define OSSMIO_MPRM
Definition XrdOssMio.hh:42
#define fstat(a, b)
Definition XrdPosix.hh:57
#define stat(a, b)
Definition XrdPosix.hh:96
struct myOpts opts
static XrdOssMioFile * Map(char *path, int fd, int opts)
Definition XrdOssMio.cc:94
static void Recycle(XrdOssMioFile *mp)
Definition XrdOssMio.cc:294
static void * preLoad(void *arg)
Definition XrdOssMio.cc:223
static void Set(int V_off, int V_preld, int V_check)
Definition XrdOssMio.cc:320
static void Display(XrdSysError &Eroute)
Definition XrdOssMio.cc:80
int Del(const char *KeyVal, XrdOucHash_Options opt=Hash_default)
T * Add(const char *KeyVal, T *KeyData, const int LifeTime=0, XrdOucHash_Options opt=Hash_default)
T * Find(const char *KeyVal, time_t *KeyTime=0)
static char * bin2hex(char *inbuff, int dlen, char *buff, int blen, bool sep=true)
int Emsg(const char *esfx, int ecode, const char *text1, const char *text2=0)
void Say(const char *text1, const char *text2=0, const char *txt3=0, const char *text4=0, const char *text5=0, const char *txt6=0)
void Lock(XrdSysMutex *Mutex)
static int Run(pthread_t *, void *(*proc)(void *), void *arg, int opts=0, const char *desc=0)