XRootD
Loading...
Searching...
No Matches
XrdClHttpFilePlugIn.cc
Go to the documentation of this file.
1
6
7#include <unistd.h>
8
9#include <cassert>
10
14#include "XrdCl/XrdClLog.hh"
15#include "XrdCl/XrdClStatus.hh"
16
17#include "XrdOuc/XrdOucCRC.hh"
18
19namespace {
20
21int MakePosixOpenFlags(XrdCl::OpenFlags::Flags flags) {
22 int posix_flags = 0;
23 if (flags & XrdCl::OpenFlags::New) {
24 posix_flags |= O_CREAT | O_EXCL;
25 }
26 if (flags & XrdCl::OpenFlags::Delete) {
27 posix_flags |= O_CREAT | O_TRUNC;
28 }
29 if (flags & XrdCl::OpenFlags::Read) {
30 posix_flags |= O_RDONLY;
31 }
32 if (flags & XrdCl::OpenFlags::Write) {
33 posix_flags |= O_WRONLY;
34 }
35 if (flags & XrdCl::OpenFlags::Update) {
36 posix_flags |= O_RDWR;
37 }
38 return posix_flags;
39}
40
41} // namespace
42
43namespace XrdCl {
44
45Davix::Context *root_davix_context_ = NULL;
46Davix::DavPosix *root_davix_client_file_ = NULL;
47
49 : davix_fd_(nullptr),
50 curr_offset(0),
51 is_open_(false),
52 filesize(0),
53 url_(),
54 properties_(),
55 logger_(DefaultEnv::GetLog()) {
56 SetUpLogging(logger_);
57 logger_->Debug(kLogXrdClHttp, "HttpFilePlugin constructed.");
58
59 std::string origin = getenv("XRDXROOTD_PROXY")? getenv("XRDXROOTD_PROXY") : "";
60 if ( origin.empty() || origin.find("=") == 0) {
61 davix_context_ = new Davix::Context();
62 davix_client_ = new Davix::DavPosix(davix_context_);
63 }
64 else {
65 if (root_davix_context_ == NULL) {
66 root_davix_context_ = new Davix::Context();
67 root_davix_client_file_ = new Davix::DavPosix(root_davix_context_);
68 }
69 davix_context_ = root_davix_context_;
70 davix_client_ = root_davix_client_file_;
71 }
72
73}
74
76 if (root_davix_context_ == NULL) {
77 delete davix_client_;
78 delete davix_context_;
79 }
80}
81
82XRootDStatus HttpFilePlugIn::Open(const std::string &url,
83 OpenFlags::Flags flags, Access::Mode /*mode*/,
84 ResponseHandler *handler, uint16_t timeout) {
85 if (is_open_) {
86 logger_->Error(kLogXrdClHttp, "URL %s already open", url.c_str());
88 }
89
90 if (XrdCl::URL(url).GetProtocol().find("https") == 0)
91 isChannelEncrypted = true;
92 else
93 isChannelEncrypted = false;
94
95 avoid_pread_ = false;
96 if (getenv(HTTP_FILE_PLUG_IN_AVOIDRANGE_ENV) != NULL)
97 avoid_pread_ = true;
98 else {
100 auto search = CGIs.find(HTTP_FILE_PLUG_IN_AVOIDRANGE_CGI);
101 if (search != CGIs.end())
102 avoid_pread_ = true;
103 }
104
105 Davix::RequestParams params;
106 if (timeout != 0) {
107 struct timespec ts = {timeout, 0};
108 params.setOperationTimeout(&ts);
109 }
110
112 auto full_path = XrdCl::URL(url).GetLocation();
113 auto pos = full_path.find_last_of('/');
114 auto base_dir =
115 pos != std::string::npos ? full_path.substr(0, pos) : full_path;
116 auto mkdir_status =
117 Posix::MkDir(*davix_client_, base_dir, XrdCl::MkDirFlags::MakePath,
118 XrdCl::Access::None, timeout);
119 if (mkdir_status.IsError()) {
120 logger_->Error(kLogXrdClHttp,
121 "Could not create parent directories when opening: %s",
122 url.c_str());
123 return mkdir_status;
124 }
125 }
126
127 if (((flags & OpenFlags::Write) || (flags & OpenFlags::Update)) &&
128 (flags & OpenFlags::Delete)) {
129 auto stat_info = new StatInfo();
130 auto status = Posix::Stat(*davix_client_, url, timeout, stat_info);
131 if (status.IsOK()) {
132 auto unlink_status = Posix::Unlink(*davix_client_, url, timeout);
133 if (unlink_status.IsError()) {
134 logger_->Error(
136 "Could not delete existing destination file: %s. Error: %s",
137 url.c_str(), unlink_status.GetErrorMessage().c_str());
138 return unlink_status;
139 }
140 }
141 delete stat_info;
142 }
143 else if (flags & OpenFlags::Read) {
144 auto stat_info = new StatInfo();
145 auto status = Posix::Stat(*davix_client_, url, timeout, stat_info);
146 if (status.IsOK()) {
147 filesize = stat_info->GetSize();
148 }
149 delete stat_info;
150 }
151
152 auto posix_open_flags = MakePosixOpenFlags(flags);
153
154 logger_->Debug(kLogXrdClHttp,
155 "Open: URL: %s, XRootD flags: %d, POSIX flags: %d",
156 url.c_str(), flags, posix_open_flags);
157
158 // res == std::pair<fd, XRootDStatus>
159 auto res = Posix::Open(*davix_client_, url, posix_open_flags, timeout);
160 if (!res.first) {
161 logger_->Error(kLogXrdClHttp, "Could not open: %s, error: %s", url.c_str(),
162 res.second.ToStr().c_str());
163 return res.second;
164 }
165
166 davix_fd_ = res.first;
167
168 logger_->Debug(kLogXrdClHttp, "Opened: %s", url.c_str());
169
170 is_open_ = true;
171 url_ = url;
172
173 auto status = new XRootDStatus();
174 handler->HandleResponse(status, nullptr);
175
176 return XRootDStatus();
177}
178
180 uint16_t /*timeout*/) {
181 if (!is_open_) {
182 logger_->Error(kLogXrdClHttp,
183 "Cannot close. URL hasn't been previously opened");
185 }
186
187 logger_->Debug(kLogXrdClHttp, "Closing davix fd: %ld", davix_fd_);
188
189 auto status = Posix::Close(*davix_client_, davix_fd_);
190 if (status.IsError()) {
191 logger_->Error(kLogXrdClHttp, "Could not close davix fd: %ld, error: %s",
192 davix_fd_, status.ToStr().c_str());
193 return status;
194 }
195
196 is_open_ = false;
197 url_.clear();
198
199 handler->HandleResponse(new XRootDStatus(), nullptr);
200
201 return XRootDStatus();
202}
203
205 uint16_t timeout) {
206 if (!is_open_) {
207 logger_->Error(kLogXrdClHttp,
208 "Cannot stat. URL hasn't been previously opened");
210 }
211
212 auto stat_info = new StatInfo();
213 auto status = Posix::Stat(*davix_client_, url_, timeout, stat_info);
214 // A file that is_open_ = true should not retune 400/3011. the only time this
215 // happen is a newly created file. Davix doesn't issue a http PUT so this file
216 // won't show up for Stat(). Here we fake a response.
217 if (status.IsError() && status.code == 400 && status.errNo == 3011) {
218 std::ostringstream data;
219 data << 140737018595560 << " " << filesize << " " << 33261 << " " << time(NULL);
220 stat_info->ParseServerResponse(data.str().c_str());
221 }
222 else if (status.IsError()) {
223 logger_->Error(kLogXrdClHttp, "Stat failed: %s", status.ToStr().c_str());
224 return status;
225 }
226
227 logger_->Debug(kLogXrdClHttp, "Stat-ed URL: %s", url_.c_str());
228
229 auto obj = new AnyObject();
230 obj->Set(stat_info);
231
232 handler->HandleResponse(new XRootDStatus(), obj);
233
234 return XRootDStatus();
235}
236
237XRootDStatus HttpFilePlugIn::Read(uint64_t offset, uint32_t size, void *buffer,
238 ResponseHandler *handler,
239 uint16_t /*timeout*/) {
240 if (!is_open_) {
241 logger_->Error(kLogXrdClHttp,
242 "Cannot read. URL hasn't previously been opened");
244 }
245
246 // DavPosix::pread will return -1 if the pread goes beyond the file size
247 size = (offset + size > filesize)? filesize - offset : size;
248 std::pair<int, XRootDStatus> res;
249 if (! avoid_pread_) {
250 res = Posix::PRead(*davix_client_, davix_fd_, buffer, size, offset);
251 }
252 else {
253 offset_locker.lock();
254 if (offset == curr_offset) {
255 res = Posix::Read(*davix_client_, davix_fd_, buffer, size);
256 }
257 else {
258 res = Posix::PRead(*davix_client_, davix_fd_, buffer, size, offset);
259 }
260 }
261
262 if (res.second.IsError()) {
263 logger_->Error(kLogXrdClHttp, "Could not read URL: %s, error: %s",
264 url_.c_str(), res.second.ToStr().c_str());
265 if (avoid_pread_) offset_locker.unlock();
266 return res.second;
267 }
268
269 int num_bytes_read = res.first;
270 curr_offset = offset + num_bytes_read;
271 if (avoid_pread_) offset_locker.unlock();
272
273 logger_->Debug(kLogXrdClHttp, "Read %d bytes, at offset %d, from URL: %s",
274 num_bytes_read, offset, url_.c_str());
275
276 auto status = new XRootDStatus();
277 auto chunk_info = new ChunkInfo(offset, num_bytes_read, buffer);
278 auto obj = new AnyObject();
279 obj->Set(chunk_info);
280 handler->HandleResponse(status, obj);
281
282 return XRootDStatus();
283}
284
286 private:
287 XrdCl::ResponseHandler *realHandler;
288 bool isChannelEncrypted;
289 public:
290 // constructor
292 bool isHttps) : realHandler(a), isChannelEncrypted(isHttps) {}
293
294 // Response Handler
296 XrdCl::AnyObject *rdresp) {
297
298 if( !status->IsOK() )
299 {
300 realHandler->HandleResponse( status, rdresp );
301 delete this;
302 return;
303 }
304
305 //using namespace XrdCl;
306
307 ChunkInfo *chunk = 0;
308 rdresp->Get(chunk);
309
310 std::vector<uint32_t> cksums;
311 if( isChannelEncrypted )
312 {
313 size_t nbpages = chunk->length / XrdSys::PageSize;
314 if( chunk->length % XrdSys::PageSize )
315 ++nbpages;
316 cksums.reserve( nbpages );
317
318 size_t size = chunk->length;
319 char *buffer = reinterpret_cast<char*>( chunk->buffer );
320
321 for( size_t pg = 0; pg < nbpages; ++pg )
322 {
323 size_t pgsize = XrdSys::PageSize;
324 if( pgsize > size ) pgsize = size;
325 uint32_t crcval = XrdOucCRC::Calc32C( buffer, pgsize );
326 cksums.push_back( crcval );
327 buffer += pgsize;
328 size -= pgsize;
329 }
330 }
331
332 PageInfo *pages = new PageInfo(chunk->offset, chunk->length, chunk->buffer, std::move(cksums));
333 delete rdresp;
334 AnyObject *response = new AnyObject();
335 response->Set( pages );
336 realHandler->HandleResponse( status, response );
337
338 delete this;
339 }
340};
341
342XRootDStatus HttpFilePlugIn::PgRead(uint64_t offset, uint32_t size, void *buffer,
343 ResponseHandler *handler,
344 uint16_t timeout) {
345 ResponseHandler *substitHandler = new PgReadSubstitutionHandler( handler, isChannelEncrypted );
346 XRootDStatus st = Read(offset, size, buffer, substitHandler, timeout);
347 return st;
348}
349
350XRootDStatus HttpFilePlugIn::Write(uint64_t offset, uint32_t size,
351 const void *buffer, ResponseHandler *handler,
352 uint16_t timeout) {
353 if (!is_open_) {
354 logger_->Error(kLogXrdClHttp,
355 "Cannot write. URL hasn't previously been opened");
357 }
358
359 // res == std::pair<int, XRootDStatus>
360 auto res =
361 Posix::PWrite(*davix_client_, davix_fd_, offset, size, buffer, timeout);
362 if (res.second.IsError()) {
363 logger_->Error(kLogXrdClHttp, "Could not write URL: %s, error: %s",
364 url_.c_str(), res.second.ToStr().c_str());
365 return res.second;
366 }
367 else
368 filesize += res.first;
369
370 logger_->Debug(kLogXrdClHttp, "Wrote %d bytes, at offset %d, to URL: %s",
371 res.first, offset, url_.c_str());
372
373 handler->HandleResponse(new XRootDStatus(), nullptr);
374
375 return XRootDStatus();
376}
377
378//------------------------------------------------------------------------
380//------------------------------------------------------------------------
382 uint32_t size,
383 const void *buffer,
384 std::vector<uint32_t> &cksums,
385 ResponseHandler *handler,
386 uint16_t timeout )
387{ (void)cksums;
388 return Write(offset, size, buffer, handler, timeout);
389}
390
392 (void)handler;
393 (void)timeout;
394
395 logger_->Debug(kLogXrdClHttp, "Sync is a no-op for HTTP.");
396
397 return XRootDStatus();
398}
399
400
402 ResponseHandler *handler,
403 uint16_t /*timeout*/) {
404 if (!is_open_) {
405 logger_->Error(kLogXrdClHttp,
406 "Cannot read. URL hasn't previously been opened");
408 }
409
410 const auto num_chunks = chunks.size();
411 std::vector<Davix::DavIOVecInput> input_vector(num_chunks);
412 std::vector<Davix::DavIOVecOuput> output_vector(num_chunks);
413
414 for (size_t i = 0; i < num_chunks; ++i) {
415 input_vector[i].diov_offset = chunks[i].offset;
416 input_vector[i].diov_size = chunks[i].length;
417 input_vector[i].diov_buffer = chunks[i].buffer;
418 }
419
420 // res == std::pair<int, XRootDStatus>
421 auto res = Posix::PReadVec(*davix_client_, davix_fd_, chunks, buffer);
422 if (res.second.IsError()) {
423 logger_->Error(kLogXrdClHttp, "Could not vectorRead URL: %s, error: %s",
424 url_.c_str(), res.second.ToStr().c_str());
425 return res.second;
426 }
427
428 int num_bytes_read = res.first;
429
430 logger_->Debug(kLogXrdClHttp, "VecRead %d bytes, from URL: %s",
431 num_bytes_read, url_.c_str());
432
433 char *output = static_cast<char *>(buffer);
434 for (size_t i = 0; i < num_chunks; ++i) {
435 std::memcpy(output + input_vector[i].diov_offset,
436 output_vector[i].diov_buffer, output_vector[i].diov_size);
437 }
438
439 auto status = new XRootDStatus();
440 auto read_info = new VectorReadInfo();
441 read_info->SetSize(num_bytes_read);
442 read_info->GetChunks() = chunks;
443 auto obj = new AnyObject();
444 obj->Set(read_info);
445 handler->HandleResponse(status, obj);
446
447 return XRootDStatus();
448}
449
450bool HttpFilePlugIn::IsOpen() const { return is_open_; }
451
452bool HttpFilePlugIn::SetProperty(const std::string &name,
453 const std::string &value) {
454 properties_[name] = value;
455 return true;
456}
457
458bool HttpFilePlugIn::GetProperty(const std::string &name,
459 std::string &value) const {
460 const auto p = properties_.find(name);
461 if (p == std::end(properties_)) {
462 return false;
463 }
464
465 value = p->second;
466 return true;
467}
468
469} // namespace XrdCl
#define HTTP_FILE_PLUG_IN_AVOIDRANGE_ENV
#define HTTP_FILE_PLUG_IN_AVOIDRANGE_CGI
void Set(Type object, bool own=true)
void Get(Type &object)
Retrieve the object being held.
virtual XRootDStatus PgRead(uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout) override
virtual XRootDStatus PgWrite(uint64_t offset, uint32_t size, const void *buffer, std::vector< uint32_t > &cksums, ResponseHandler *handler, uint16_t timeout) override
virtual XRootDStatus Sync(ResponseHandler *handler, uint16_t timeout) override
virtual XRootDStatus Stat(bool force, ResponseHandler *handler, uint16_t timeout) override
virtual XRootDStatus Write(uint64_t offset, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout) override
virtual XRootDStatus Read(uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout) override
virtual XRootDStatus VectorRead(const ChunkList &chunks, void *buffer, XrdCl::ResponseHandler *handler, uint16_t timeout) override
virtual ~HttpFilePlugIn() noexcept
virtual XRootDStatus Open(const std::string &url, OpenFlags::Flags flags, Access::Mode mode, ResponseHandler *handler, uint16_t timeout) override
virtual bool GetProperty(const std::string &name, std::string &value) const override
virtual XRootDStatus Close(ResponseHandler *handler, uint16_t timeout) override
virtual bool SetProperty(const std::string &name, const std::string &value) override
virtual bool IsOpen() const override
void Error(uint64_t topic, const char *format,...)
Report an error.
Definition XrdClLog.cc:231
void Debug(uint64_t topic, const char *format,...)
Print a debug message.
Definition XrdClLog.cc:282
void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *rdresp)
PgReadSubstitutionHandler(XrdCl::ResponseHandler *a, bool isHttps)
Handle an async response.
virtual void HandleResponse(XRootDStatus *status, AnyObject *response)
Object stat info.
URL representation.
Definition XrdClURL.hh:31
std::map< std::string, std::string > ParamsMap
Definition XrdClURL.hh:33
std::string GetLocation() const
Get location (protocol://host:port/path)
Definition XrdClURL.cc:330
const ParamsMap & GetParams() const
Get the URL params.
Definition XrdClURL.hh:239
static uint32_t Calc32C(const void *data, size_t count, uint32_t prevcs=0)
Definition XrdOucCRC.cc:190
std::pair< int, XrdCl::XRootDStatus > PReadVec(Davix::DavPosix &davix_client, DAVIX_FD *fd, const XrdCl::ChunkList &chunks, void *buffer)
std::pair< int, XrdCl::XRootDStatus > PWrite(Davix::DavPosix &davix_client, DAVIX_FD *fd, uint64_t offset, uint32_t size, const void *buffer, uint16_t timeout)
std::pair< int, XRootDStatus > PRead(Davix::DavPosix &davix_client, DAVIX_FD *fd, void *buffer, uint32_t size, uint64_t offset)
XRootDStatus Unlink(Davix::DavPosix &davix_client, const std::string &url, uint16_t timeout)
ReadImpl< false > Read(Ctx< File > file, Arg< uint64_t > offset, Arg< uint32_t > size, Arg< void * > buffer, uint16_t timeout=0)
Factory for creating ReadImpl objects.
MkDirImpl< false > MkDir
const uint16_t stError
An error occurred that could potentially be retried.
StatImpl< false > Stat(Ctx< File > file, Arg< bool > force, uint16_t timeout=0)
CloseImpl< false > Close(Ctx< File > file, uint16_t timeout=0)
Factory for creating CloseImpl objects.
const uint16_t errInvalidOp
Davix::Context * root_davix_context_
std::vector< ChunkInfo > ChunkList
List of chunks.
Davix::DavPosix * root_davix_client_file_
void SetUpLogging(Log *logger)
static const uint64_t kLogXrdClHttp
OpenImpl< false > Open(Ctx< File > file, Arg< std::string > url, Arg< OpenFlags::Flags > flags, Arg< Access::Mode > mode=Access::None, uint16_t timeout=0)
Factory for creating ReadImpl objects.
static const int PageSize
Describe a data chunk for vector read.
void * buffer
length of the chunk
uint32_t length
offset in the file
@ MakePath
create the entire directory tree if it doesn't exist
Flags
Open flags, may be or'd when appropriate.
@ Read
Open only for reading.
@ Write
Open only for writing.
@ Update
Open for reading and writing.
bool IsOK() const
We're fine.