GDAL

cpl_vsil_curl_class.h
1 /******************************************************************************
2 *
3 * Project: CPL - Common Portability Library
4 * Purpose: Declarations for /vsicurl/ and related file systems
5 * Author: Even Rouault, even.rouault at spatialys.com
6 *
7 ******************************************************************************
8 * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the "Software"),
12 * to deal in the Software without restriction, including without limitation
13 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 * and/or sell copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included
18 * in all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 ****************************************************************************/
28
29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
31
32 #ifdef HAVE_CURL
33
34 #include "cpl_aws.h"
35 #include "cpl_azure.h"
36 #include "cpl_port.h"
37 #include "cpl_json.h"
38 #include "cpl_string.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
41
42 #include "cpl_curl_priv.h"
43
44 #include <set>
45 #include <map>
46 #include <memory>
47 #include <mutex>
48
50
51 // Leave it for backward compatibility, but deprecate.
52 #define HAVE_CURLINFO_REDIRECT_URL
53
54 void VSICurlStreamingClearCache( void ); // from cpl_vsil_curl_streaming.cpp
55
56 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL,
57 const char * const* papszOptions);
58 struct curl_slist* VSICurlMergeHeaders( struct curl_slist* poDest,
59 struct curl_slist* poSrcToDestroy );
60
61 struct curl_slist* VSICurlSetContentTypeFromExt(struct curl_slist* polist,
62 const char *pszPath);
63
64 struct curl_slist* VSICurlSetCreationHeadersFromOptions(struct curl_slist* headers,
65 CSLConstList papszOptions,
66 const char *pszPath);
67
68 namespace cpl {
69
70 typedef enum
71 {
72 EXIST_UNKNOWN = -1,
73 EXIST_NO,
74 EXIST_YES,
75 } ExistStatus;
76
77 class FileProp
78 {
79 public:
80 unsigned int nGenerationAuthParameters = 0;
81 ExistStatus eExists = EXIST_UNKNOWN;
82 vsi_l_offset fileSize = 0;
83 time_t mTime = 0;
84 time_t nExpireTimestampLocal = 0;
85 CPLString osRedirectURL{};
86 bool bHasComputedFileSize = false;
87 bool bIsDirectory = false;
88 int nMode = 0; // st_mode member of struct stat
89 bool bS3LikeRedirect = false;
90 CPLString ETag{};
91 };
92
93 struct CachedDirList
94 {
95 bool bGotFileList = false;
96 unsigned int nGenerationAuthParameters = 0;
97 CPLStringList oFileList{}; /* only file name without path */
98 };
99
100 struct WriteFuncStruct
101 {
102 char* pBuffer = nullptr;
103 size_t nSize = 0;
104 bool bIsHTTP = false;
105 bool bMultiRange = false;
106 vsi_l_offset nStartOffset = 0;
107 vsi_l_offset nEndOffset = 0;
108 int nHTTPCode = 0;
109 vsi_l_offset nContentLength = 0;
110 bool bFoundContentRange = false;
111 bool bError = false;
112 bool bInterruptDownload = false;
113 bool bDetectRangeDownloadingError = false;
114 GIntBig nTimestampDate = 0; // Corresponds to Date: header field
115
116 VSILFILE *fp = nullptr;
117 VSICurlReadCbkFunc pfnReadCbk = nullptr;
118 void *pReadCbkUserData = nullptr;
119 bool bInterrupted = false;
120
121 #if !CURL_AT_LEAST_VERSION(7,54,0)
122 // Workaround to ignore extra HTTP response headers from
123 // proxies in older versions of curl.
124 // CURLOPT_SUPPRESS_CONNECT_HEADERS fixes this
125 bool bIsProxyConnectHeader = false;
126 #endif
127 };
128
129 struct PutData
130 {
131 const GByte* pabyData = nullptr;
132 size_t nOff = 0;
133 size_t nTotalSize = 0;
134
135 static size_t ReadCallBackBuffer( char *buffer, size_t size,
136 size_t nitems, void *instream )
137 {
138 PutData* poThis = static_cast<PutData *>(instream);
139 const size_t nSizeMax = size * nitems;
140 const size_t nSizeToWrite =
141 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
142 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
143 poThis->nOff += nSizeToWrite;
144 return nSizeToWrite;
145 }
146 };
147
148 /************************************************************************/
149 /* VSICurlFilesystemHandler */
150 /************************************************************************/
151
152 class VSICurlHandle;
153
154 class VSICurlFilesystemHandlerBase : public VSIFilesystemHandler
155 {
156 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBase)
157
158 struct FilenameOffsetPair
159 {
160 std::string filename_;
161 vsi_l_offset offset_;
162
163 FilenameOffsetPair(const std::string& filename,
164 vsi_l_offset offset) :
165 filename_(filename), offset_(offset) {}
166
167 bool operator==(const FilenameOffsetPair& other) const
168 {
169 return filename_ == other.filename_ &&
170 offset_ == other.offset_;
171 }
172 };
173 struct FilenameOffsetPairHasher
174 {
175 std::size_t operator()(const FilenameOffsetPair& k) const
176 {
177 return std::hash<std::string>()(k.filename_) ^
178 std::hash<vsi_l_offset>()(k.offset_);
179 }
180 };
181
182 using RegionCacheType =
183 lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
184 lru11::NullLock,
185 std::unordered_map<
186 FilenameOffsetPair,
187 typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
188 std::shared_ptr<std::string>>>::iterator,
189 FilenameOffsetPairHasher>>;
190
191 std::unique_ptr<RegionCacheType> m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use GetRegionCache();
192 RegionCacheType* GetRegionCache();
193
194 lru11::Cache<std::string, FileProp> oCacheFileProp;
195
196 int nCachedFilesInDirList = 0;
197 lru11::Cache<std::string, CachedDirList> oCacheDirList;
198
199 char** ParseHTMLFileList(const char* pszFilename,
200 int nMaxFiles,
201 char* pszData,
202 bool* pbGotFileList);
203
204 protected:
205 CPLMutex *hMutex = nullptr;
206
207 virtual VSICurlHandle* CreateFileHandle(const char* pszFilename);
208 virtual char** GetFileList(const char *pszFilename,
209 int nMaxFiles,
210 bool* pbGotFileList);
211
212 void RegisterEmptyDir( const CPLString& osDirname );
213
214 bool AnalyseS3FileList( const CPLString& osBaseURL,
215 const char* pszXML,
216 CPLStringList& osFileList,
217 int nMaxFiles,
218 bool bIgnoreGlacierStorageClass,
219 bool& bIsTruncated );
220
221 void AnalyseSwiftFileList( const CPLString& osBaseURL,
222 const CPLString& osPrefix,
223 const char* pszJson,
224 CPLStringList& osFileList,
225 int nMaxFilesThisQuery,
226 int nMaxFiles,
227 bool& bIsTruncated,
228 CPLString& osNextMarker );
229
230 static const char* GetOptionsStatic();
231
232 static bool IsAllowedFilename( const char* pszFilename );
233
234 VSICurlFilesystemHandlerBase();
235
236 public:
237 ~VSICurlFilesystemHandlerBase() override;
238
239 VSIVirtualHandle *Open( const char *pszFilename,
240 const char *pszAccess,
241 bool bSetError,
242 CSLConstList /* papszOptions */ ) override;
243
244 int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
245 int nFlags ) override;
246 int Unlink( const char *pszFilename ) override;
247 int Rename( const char *oldpath, const char *newpath ) override;
248 int Mkdir( const char *pszDirname, long nMode ) override;
249 int Rmdir( const char *pszDirname ) override;
250 char **ReadDir( const char *pszDirname ) override
251 { return ReadDirEx(pszDirname, 0); }
252 char **ReadDirEx( const char *pszDirname, int nMaxFiles ) override;
253 char **SiblingFiles( const char *pszFilename ) override;
254
255 int HasOptimizedReadMultiRange( const char* /* pszPath */ )
256 override { return true; }
257
258 const char* GetActualURL(const char* pszFilename) override;
259
260 const char* GetOptions() override;
261
262 char** GetFileMetadata( const char * pszFilename, const char* pszDomain,
263 CSLConstList papszOptions ) override;
264
265 char **ReadDirInternal( const char *pszDirname, int nMaxFiles,
266 bool* pbGotFileList );
267 void InvalidateDirContent( const char *pszDirname );
268
269 virtual const char* GetDebugKey() const = 0;
270
271 virtual CPLString GetFSPrefix() const = 0;
272 virtual bool AllowCachedDataFor(const char* pszFilename);
273
274 std::shared_ptr<std::string> GetRegion( const char* pszURL,
275 vsi_l_offset nFileOffsetStart );
276
277 void AddRegion( const char* pszURL,
278 vsi_l_offset nFileOffsetStart,
279 size_t nSize,
280 const char *pData );
281
282 bool GetCachedFileProp( const char* pszURL,
283 FileProp& oFileProp );
284 void SetCachedFileProp( const char* pszURL,
285 FileProp& oFileProp );
286 void InvalidateCachedData( const char* pszURL );
287
288 CURLM *GetCurlMultiHandleFor( const CPLString& osURL );
289
290 virtual void ClearCache();
291 virtual void PartialClearCache(const char* pszFilename);
292
293
294 bool GetCachedDirList( const char* pszURL,
295 CachedDirList& oCachedDirList );
296 void SetCachedDirList( const char* pszURL,
297 CachedDirList& oCachedDirList );
298 bool ExistsInCacheDirList( const CPLString& osDirname, bool *pbIsDir );
299
300 virtual CPLString GetURLFromFilename( const CPLString& osFilename );
301
302 std::string GetStreamingFilename(const std::string& osFilename) const override = 0;
303 };
304
305
306 class VSICurlFilesystemHandler: public VSICurlFilesystemHandlerBase
307 {
308 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
309
310 public:
311 VSICurlFilesystemHandler() = default;
312
313 const char* GetDebugKey() const override { return "VSICURL"; }
314
315 CPLString GetFSPrefix() const override { return "/vsicurl/"; }
316
317 std::string GetStreamingFilename(const std::string& osFilename) const override;
318 };
319
320 /************************************************************************/
321 /* VSICurlHandle */
322 /************************************************************************/
323
324 class VSICurlHandle : public VSIVirtualHandle
325 {
326 CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
327
328 protected:
329 VSICurlFilesystemHandlerBase* poFS = nullptr;
330
331 bool m_bCached = true;
332
333 FileProp oFileProp{};
334
335 CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
336 char* m_pszURL = nullptr; // e.g "http://example.com/foo"
337 std::string m_osQueryString{}; // e.g. an Azure SAS
338
339 char **m_papszHTTPOptions = nullptr;
340
341 vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
342 int nBlocksToDownload = 1;
343
344 bool bStopOnInterruptUntilUninstall = false;
345 bool bInterrupted = false;
346 VSICurlReadCbkFunc pfnReadCbk = nullptr;
347 void *pReadCbkUserData = nullptr;
348
349 int m_nMaxRetry = 0;
350 double m_dfRetryDelay = 0.0;
351
352 CPLStringList m_aosHeaders{};
353
354 void DownloadRegionPostProcess( const vsi_l_offset startOffset,
355 const int nBlocks,
356 const char* pBuffer,
357 size_t nSize );
358
359 private:
360
361 vsi_l_offset curOffset = 0;
362
363 bool bEOF = false;
364
365 virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
366
367 bool m_bUseHead = false;
368 bool m_bUseRedirectURLIfNoQueryStringParams = false;
369
370 int ReadMultiRangeSingleGet( int nRanges, void ** ppData,
371 const vsi_l_offset* panOffsets,
372 const size_t* panSizes );
373 CPLString GetRedirectURLIfValid(bool& bHasExpired);
374
375 protected:
376 virtual struct curl_slist* GetCurlHeaders( const CPLString& /*osVerb*/,
377 const struct curl_slist* /* psExistingHeaders */)
378 { return nullptr; }
379 virtual bool AllowAutomaticRedirection() { return true; }
380 virtual bool CanRestartOnError( const char*, const char*, bool ) { return false; }
381 virtual bool UseLimitRangeGetInsteadOfHead() { return false; }
382 virtual bool IsDirectoryFromExists( const char* /*pszVerb*/, int /*response_code*/ ) { return false; }
383 virtual void ProcessGetFileSizeResult(const char* /* pszContent */ ) {}
384 void SetURL(const char* pszURL);
385 virtual bool Authenticate() { return false; }
386
387 public:
388
389 VSICurlHandle( VSICurlFilesystemHandlerBase* poFS,
390 const char* pszFilename,
391 const char* pszURLIn = nullptr );
392 ~VSICurlHandle() override;
393
394 int Seek( vsi_l_offset nOffset, int nWhence ) override;
395 vsi_l_offset Tell() override;
396 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
397 int ReadMultiRange( int nRanges, void ** ppData,
398 const vsi_l_offset* panOffsets,
399 const size_t* panSizes ) override;
400 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
401 int Eof() override;
402 int Flush() override;
403 int Close() override;
404
405 bool IsKnownFileSize() const { return oFileProp.bHasComputedFileSize; }
406 vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
407 virtual vsi_l_offset GetFileSize( bool bSetError ) { return GetFileSizeOrHeaders(bSetError, false); }
408 bool Exists( bool bSetError );
409 bool IsDirectory() const { return oFileProp.bIsDirectory; }
410 int GetMode() const { return oFileProp.nMode; }
411 time_t GetMTime() const { return oFileProp.mTime; }
412 const CPLStringList& GetHeaders() { return m_aosHeaders; }
413
414 int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
415 void* pfnUserData,
416 int bStopOnInterruptUntilUninstall );
417 int UninstallReadCbk();
418
419 const char *GetURL() const { return m_pszURL; }
420 };
421
422 /************************************************************************/
423 /* IVSIS3LikeFSHandler */
424 /************************************************************************/
425
426 class IVSIS3LikeFSHandler: public VSICurlFilesystemHandlerBase
427 {
428 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
429
430 bool CopyFile(VSILFILE* fpIn,
431 vsi_l_offset nSourceSize,
432 const char* pszSource,
433 const char* pszTarget,
434 GDALProgressFunc pProgressFunc,
435 void *pProgressData);
436 virtual int MkdirInternal( const char *pszDirname, long nMode, bool bDoStatCheck );
437
438 protected:
439 char** GetFileList( const char *pszFilename,
440 int nMaxFiles,
441 bool* pbGotFileList ) override;
442
443 virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
444 const char* pszURI, bool bAllowNoObject) = 0;
445
446 virtual int CopyObject( const char *oldpath, const char *newpath,
447 CSLConstList papszMetadata );
448
449 int RmdirRecursiveInternal( const char* pszDirname, int nBatchSize);
450
451 IVSIS3LikeFSHandler() = default;
452
453 public:
454 int Unlink( const char *pszFilename ) override;
455 int Mkdir( const char *pszDirname, long nMode ) override;
456 int Rmdir( const char *pszDirname ) override;
457 int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
458 int nFlags ) override;
459 int Rename( const char *oldpath, const char *newpath ) override;
460
461 virtual int DeleteObject( const char *pszFilename );
462
463 virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
464 virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
465
466 bool Sync( const char* pszSource, const char* pszTarget,
467 const char* const * papszOptions,
468 GDALProgressFunc pProgressFunc,
469 void *pProgressData,
470 char*** ppapszOutputs ) override;
471
472 VSIDIR* OpenDir( const char *pszPath, int nRecurseDepth,
473 const char* const *papszOptions) override;
474
475 // Multipart upload
476 virtual bool SupportsParallelMultipartUpload() const { return false; }
477
478 virtual CPLString InitiateMultipartUpload(
479 const std::string& osFilename,
480 IVSIS3LikeHandleHelper *poS3HandleHelper,
481 int nMaxRetry,
482 double dfRetryDelay,
483 CSLConstList papszOptions);
484 virtual CPLString UploadPart(const CPLString& osFilename,
485 int nPartNumber,
486 const std::string& osUploadID,
487 vsi_l_offset nPosition,
488 const void* pabyBuffer,
489 size_t nBufferSize,
490 IVSIS3LikeHandleHelper *poS3HandleHelper,
491 int nMaxRetry,
492 double dfRetryDelay);
493 virtual bool CompleteMultipart(const CPLString& osFilename,
494 const CPLString& osUploadID,
495 const std::vector<CPLString>& aosEtags,
496 vsi_l_offset nTotalSize,
497 IVSIS3LikeHandleHelper *poS3HandleHelper,
498 int nMaxRetry,
499 double dfRetryDelay);
500 virtual bool AbortMultipart(const CPLString& osFilename,
501 const CPLString& osUploadID,
502 IVSIS3LikeHandleHelper *poS3HandleHelper,
503 int nMaxRetry,
504 double dfRetryDelay);
505
506 bool AbortPendingUploads(const char* pszFilename) override;
507 };
508
509 /************************************************************************/
510 /* IVSIS3LikeHandle */
511 /************************************************************************/
512
513 class IVSIS3LikeHandle: public VSICurlHandle
514 {
515 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
516
517 protected:
518 bool UseLimitRangeGetInsteadOfHead() override { return true; }
519 bool IsDirectoryFromExists( const char* pszVerb,
520 int response_code ) override
521 {
522 // A bit dirty, but on S3, a GET on a existing directory returns a 416
523 return response_code == 416 && EQUAL(pszVerb, "GET") &&
524 CPLString(m_pszURL).back() == '/';
525 }
526 void ProcessGetFileSizeResult( const char* pszContent ) override
527 {
528 oFileProp.bIsDirectory = strstr(pszContent, "ListBucketResult") != nullptr;
529 }
530
531 public:
532 IVSIS3LikeHandle( VSICurlFilesystemHandlerBase* poFSIn,
533 const char* pszFilename,
534 const char* pszURLIn ) :
535 VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
536 ~IVSIS3LikeHandle() override {}
537 };
538
539 /************************************************************************/
540 /* VSIS3WriteHandle */
541 /************************************************************************/
542
543 class VSIS3WriteHandle final : public VSIVirtualHandle
544 {
545 CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
546
547 IVSIS3LikeFSHandler *m_poFS = nullptr;
548 CPLString m_osFilename{};
549 IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
550 bool m_bUseChunked = false;
551 CPLStringList m_aosOptions{};
552
553 vsi_l_offset m_nCurOffset = 0;
554 int m_nBufferOff = 0;
555 int m_nBufferSize = 0;
556 bool m_bClosed = false;
557 GByte *m_pabyBuffer = nullptr;
558 CPLString m_osUploadID{};
559 int m_nPartNumber = 0;
560 std::vector<CPLString> m_aosEtags{};
561 bool m_bError = false;
562
563 CURLM *m_hCurlMulti = nullptr;
564 CURL *m_hCurl = nullptr;
565 const void *m_pBuffer = nullptr;
566 CPLString m_osCurlErrBuf{};
567 size_t m_nChunkedBufferOff = 0;
568 size_t m_nChunkedBufferSize = 0;
569 size_t m_nWrittenInPUT = 0;
570
571 int m_nMaxRetry = 0;
572 double m_dfRetryDelay = 0.0;
573 WriteFuncStruct m_sWriteFuncHeaderData{};
574
575 bool UploadPart();
576 bool DoSinglePartPUT();
577
578 static size_t ReadCallBackBufferChunked( char *buffer, size_t size,
579 size_t nitems, void *instream );
580 size_t WriteChunked( const void *pBuffer,
581 size_t nSize, size_t nMemb );
582 int FinishChunkedTransfer();
583
584 void InvalidateParentDirectory();
585
586 public:
587 VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
588 const char* pszFilename,
589 IVSIS3LikeHandleHelper* poS3HandleHelper,
590 bool bUseChunked,
591 CSLConstList papszOptions );
592 ~VSIS3WriteHandle() override;
593
594 int Seek( vsi_l_offset nOffset, int nWhence ) override;
595 vsi_l_offset Tell() override;
596 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
597 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
598 int Eof() override;
599 int Close() override;
600
601 bool IsOK() { return m_bUseChunked || m_pabyBuffer != nullptr; }
602 };
603
604 /************************************************************************/
605 /* VSIAppendWriteHandle */
606 /************************************************************************/
607
608 class VSIAppendWriteHandle : public VSIVirtualHandle
609 {
610 CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
611
612 protected:
613
614 VSICurlFilesystemHandlerBase* m_poFS = nullptr;
615 CPLString m_osFSPrefix{};
616 CPLString m_osFilename{};
617
618 vsi_l_offset m_nCurOffset = 0;
619 int m_nBufferOff = 0;
620 int m_nBufferSize = 0;
621 int m_nBufferOffReadCallback = 0;
622 bool m_bClosed = false;
623 GByte *m_pabyBuffer = nullptr;
624 bool m_bError = false;
625
626 static size_t ReadCallBackBuffer( char *buffer, size_t size,
627 size_t nitems, void *instream );
628 virtual bool Send(bool bIsLastBlock) = 0;
629
630 public:
631 VSIAppendWriteHandle( VSICurlFilesystemHandlerBase* poFS,
632 const char* pszFSPrefix,
633 const char* pszFilename,
634 int nChunkSize );
635 virtual ~VSIAppendWriteHandle();
636
637 int Seek( vsi_l_offset nOffset, int nWhence ) override;
638 vsi_l_offset Tell() override;
639 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
640 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
641 int Eof() override;
642 int Close() override;
643
644 bool IsOK() { return m_pabyBuffer != nullptr; }
645 };
646
647 /************************************************************************/
648 /* CurlRequestHelper */
649 /************************************************************************/
650
651 struct CurlRequestHelper
652 {
653 WriteFuncStruct sWriteFuncData{};
654 WriteFuncStruct sWriteFuncHeaderData{};
655 char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
656
657 CurlRequestHelper();
658 ~CurlRequestHelper();
659 long perform(CURL* hCurlHandle,
660 struct curl_slist* headers, // ownership transferred
661 VSICurlFilesystemHandlerBase *poFS,
662 IVSIS3LikeHandleHelper *poS3HandleHelper);
663 };
664
665 /************************************************************************/
666 /* NetworkStatisticsLogger */
667 /************************************************************************/
668
669 class NetworkStatisticsLogger
670 {
671 static int gnEnabled;
672 static NetworkStatisticsLogger gInstance;
673
674 NetworkStatisticsLogger() = default;
675
676 std::mutex m_mutex{};
677
678 struct Counters
679 {
680 GIntBig nHEAD = 0;
681 GIntBig nGET = 0;
682 GIntBig nPUT = 0;
683 GIntBig nPOST = 0;
684 GIntBig nDELETE = 0;
685 GIntBig nGETDownloadedBytes = 0;
686 GIntBig nPUTUploadedBytes = 0;
687 GIntBig nPOSTDownloadedBytes = 0;
688 GIntBig nPOSTUploadedBytes = 0;
689 };
690
691 enum class ContextPathType
692 {
693 FILESYSTEM,
694 FILE,
695 ACTION,
696 };
697
698 struct ContextPathItem
699 {
700 ContextPathType eType;
701 CPLString osName;
702
703 ContextPathItem(ContextPathType eTypeIn, const CPLString& osNameIn):
704 eType(eTypeIn), osName(osNameIn) {}
705
706 bool operator< (const ContextPathItem& other ) const
707 {
708 if( static_cast<int>(eType) < static_cast<int>(other.eType) )
709 return true;
710 if( static_cast<int>(eType) > static_cast<int>(other.eType) )
711 return false;
712 return osName < other.osName;
713 }
714 };
715
716 struct Stats
717 {
718 Counters counters{};
719 std::map<ContextPathItem, Stats> children{};
720
721 void AsJSON(CPLJSONObject& oJSON) const;
722 };
723
724 // Workaround bug in Coverity Scan
725 // coverity[generated_default_constructor_used_in_field_initializer]
726 Stats m_stats{};
727 std::map<GIntBig, std::vector<ContextPathItem>> m_mapThreadIdToContextPath{};
728
729 static void ReadEnabled();
730
731 std::vector<Counters*> GetCountersForContext();
732
733 public:
734
735 static inline bool IsEnabled()
736 {
737 if( gnEnabled < 0)
738 {
739 ReadEnabled();
740 }
741 return gnEnabled == TRUE;
742 }
743
744 static void EnterFileSystem(const char* pszName);
745
746 static void LeaveFileSystem();
747
748 static void EnterFile(const char* pszName);
749
750 static void LeaveFile();
751
752 static void EnterAction(const char* pszName);
753
754 static void LeaveAction();
755
756 static void LogHEAD();
757
758 static void LogGET(size_t nDownloadedBytes);
759
760 static void LogPUT(size_t nUploadedBytes);
761
762 static void LogPOST(size_t nUploadedBytes,
763 size_t nDownloadedBytes);
764
765 static void LogDELETE();
766
767 static void Reset();
768
769 static CPLString GetReportAsSerializedJSON();
770 };
771
772 struct NetworkStatisticsFileSystem
773 {
774 inline explicit NetworkStatisticsFileSystem(const char* pszName) {
775 NetworkStatisticsLogger::EnterFileSystem(pszName);
776 }
777
778 inline ~NetworkStatisticsFileSystem()
779 {
780 NetworkStatisticsLogger::LeaveFileSystem();
781 }
782 };
783
784 struct NetworkStatisticsFile
785 {
786 inline explicit NetworkStatisticsFile(const char* pszName) {
787 NetworkStatisticsLogger::EnterFile(pszName);
788 }
789
790 inline ~NetworkStatisticsFile()
791 {
792 NetworkStatisticsLogger::LeaveFile();
793 }
794 };
795
796 struct NetworkStatisticsAction
797 {
798 inline explicit NetworkStatisticsAction(const char* pszName) {
799 NetworkStatisticsLogger::EnterAction(pszName);
800 }
801
802 inline ~NetworkStatisticsAction()
803 {
804 NetworkStatisticsLogger::LeaveAction();
805 }
806 };
807
808
809 int VSICURLGetDownloadChunkSize();
810
811 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
812 VSILFILE *fp,
813 VSICurlReadCbkFunc pfnReadCbk,
814 void *pReadCbkUserData );
815 size_t VSICurlHandleWriteFunc( void *buffer, size_t count,
816 size_t nmemb, void *req );
817 void MultiPerform(CURLM* hCurlMultiHandle,
818 CURL* hEasyHandle = nullptr);
819 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
820
821 int VSICurlParseUnixPermissions(const char* pszPermissions);
822
823 } // namespace cpl
824
826
827 #endif // HAVE_CURL
828
829 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
GByte
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:215
CPLStringList
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:441
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:332
EQUAL
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:568
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1169
CPLJSONObject
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:53
cpl_string.h
cpl_json.h
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:140
VSIStatBufL
struct VSI_STAT64_T VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:195
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:244
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:323
cpl_port.h
VSIVirtualHandle
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
VSI_L_OFFSET_MAX
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:142
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:955
VSILFILE
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:156