GDAL

cpl_vsil_curl_class.h
1 /******************************************************************************
2 *
3 * Project: CPL - Common Portability Library
4 * Purpose: Declarations for /vsicurl/ and related file systems
5 * Author: Even Rouault, even.rouault at spatialys.com
6 *
7 ******************************************************************************
8 * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the "Software"),
12 * to deal in the Software without restriction, including without limitation
13 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 * and/or sell copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included
18 * in all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 ****************************************************************************/
28
29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
31
32 #ifdef HAVE_CURL
33
34 #include "cpl_aws.h"
35 #include "cpl_azure.h"
36 #include "cpl_port.h"
37 #include "cpl_json.h"
38 #include "cpl_string.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
41
42 #include "cpl_curl_priv.h"
43
44 #include <set>
45 #include <map>
46 #include <memory>
47 #include <mutex>
48
50
51 // Leave it for backward compatibility, but deprecate.
52 #define HAVE_CURLINFO_REDIRECT_URL
53
54 void VSICurlStreamingClearCache( void ); // from cpl_vsil_curl_streaming.cpp
55
56 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL,
57 const char * const* papszOptions);
58 struct curl_slist* VSICurlMergeHeaders( struct curl_slist* poDest,
59 struct curl_slist* poSrcToDestroy );
60
61 struct curl_slist* VSICurlSetContentTypeFromExt(struct curl_slist* polist,
62 const char *pszPath);
63
64 struct curl_slist* VSICurlSetCreationHeadersFromOptions(struct curl_slist* headers,
65 CSLConstList papszOptions,
66 const char *pszPath);
67
68 namespace cpl {
69
70 typedef enum
71 {
72 EXIST_UNKNOWN = -1,
73 EXIST_NO,
74 EXIST_YES,
75 } ExistStatus;
76
77 class FileProp
78 {
79 public:
80 unsigned int nGenerationAuthParameters = 0;
81 ExistStatus eExists = EXIST_UNKNOWN;
82 vsi_l_offset fileSize = 0;
83 time_t mTime = 0;
84 time_t nExpireTimestampLocal = 0;
85 CPLString osRedirectURL{};
86 bool bHasComputedFileSize = false;
87 bool bIsDirectory = false;
88 int nMode = 0; // st_mode member of struct stat
89 bool bS3LikeRedirect = false;
90 CPLString ETag{};
91 };
92
93 struct CachedDirList
94 {
95 bool bGotFileList = false;
96 unsigned int nGenerationAuthParameters = 0;
97 CPLStringList oFileList{}; /* only file name without path */
98 };
99
100 struct WriteFuncStruct
101 {
102 char* pBuffer = nullptr;
103 size_t nSize = 0;
104 bool bIsHTTP = false;
105 bool bIsInHeader = false;
106 bool bMultiRange = false;
107 vsi_l_offset nStartOffset = 0;
108 vsi_l_offset nEndOffset = 0;
109 int nHTTPCode = 0;
110 vsi_l_offset nContentLength = 0;
111 bool bFoundContentRange = false;
112 bool bError = false;
113 bool bDownloadHeaderOnly = false;
114 bool bDetectRangeDownloadingError = false;
115 GIntBig nTimestampDate = 0; // Corresponds to Date: header field
116
117 VSILFILE *fp = nullptr;
118 VSICurlReadCbkFunc pfnReadCbk = nullptr;
119 void *pReadCbkUserData = nullptr;
120 bool bInterrupted = false;
121
122 #if !CURL_AT_LEAST_VERSION(7,54,0)
123 // Workaround to ignore extra HTTP response headers from
124 // proxies in older versions of curl.
125 // CURLOPT_SUPPRESS_CONNECT_HEADERS fixes this
126 bool bIsProxyConnectHeader = false;
127 #endif
128 };
129
130 struct PutData
131 {
132 const GByte* pabyData = nullptr;
133 size_t nOff = 0;
134 size_t nTotalSize = 0;
135
136 static size_t ReadCallBackBuffer( char *buffer, size_t size,
137 size_t nitems, void *instream )
138 {
139 PutData* poThis = static_cast<PutData *>(instream);
140 const size_t nSizeMax = size * nitems;
141 const size_t nSizeToWrite =
142 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
143 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
144 poThis->nOff += nSizeToWrite;
145 return nSizeToWrite;
146 }
147 };
148
149 /************************************************************************/
150 /* VSICurlFilesystemHandler */
151 /************************************************************************/
152
153 class VSICurlHandle;
154
155 class VSICurlFilesystemHandler : public VSIFilesystemHandler
156 {
157 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
158
159 struct FilenameOffsetPair
160 {
161 std::string filename_;
162 vsi_l_offset offset_;
163
164 FilenameOffsetPair(const std::string& filename,
165 vsi_l_offset offset) :
166 filename_(filename), offset_(offset) {}
167
168 bool operator==(const FilenameOffsetPair& other) const
169 {
170 return filename_ == other.filename_ &&
171 offset_ == other.offset_;
172 }
173 };
174 struct FilenameOffsetPairHasher
175 {
176 std::size_t operator()(const FilenameOffsetPair& k) const
177 {
178 return std::hash<std::string>()(k.filename_) ^
179 std::hash<vsi_l_offset>()(k.offset_);
180 }
181 };
182
183 using RegionCacheType =
184 lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
185 lru11::NullLock,
186 std::unordered_map<
187 FilenameOffsetPair,
188 typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
189 std::shared_ptr<std::string>>>::iterator,
190 FilenameOffsetPairHasher>>;
191
192 std::unique_ptr<RegionCacheType> m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use GetRegionCache();
193 RegionCacheType* GetRegionCache();
194
195 lru11::Cache<std::string, FileProp> oCacheFileProp;
196
197 int nCachedFilesInDirList = 0;
198 lru11::Cache<std::string, CachedDirList> oCacheDirList;
199
200 char** ParseHTMLFileList(const char* pszFilename,
201 int nMaxFiles,
202 char* pszData,
203 bool* pbGotFileList);
204
205 protected:
206 CPLMutex *hMutex = nullptr;
207
208 virtual VSICurlHandle* CreateFileHandle(const char* pszFilename);
209 virtual char** GetFileList(const char *pszFilename,
210 int nMaxFiles,
211 bool* pbGotFileList);
212
213 void RegisterEmptyDir( const CPLString& osDirname );
214
215 bool AnalyseS3FileList( const CPLString& osBaseURL,
216 const char* pszXML,
217 CPLStringList& osFileList,
218 int nMaxFiles,
219 bool bIgnoreGlacierStorageClass,
220 bool& bIsTruncated );
221
222 void AnalyseSwiftFileList( const CPLString& osBaseURL,
223 const CPLString& osPrefix,
224 const char* pszJson,
225 CPLStringList& osFileList,
226 int nMaxFilesThisQuery,
227 int nMaxFiles,
228 bool& bIsTruncated,
229 CPLString& osNextMarker );
230
231 static const char* GetOptionsStatic();
232
233 static bool IsAllowedFilename( const char* pszFilename );
234
235 public:
236 VSICurlFilesystemHandler();
237 ~VSICurlFilesystemHandler() override;
238
239 VSIVirtualHandle *Open( const char *pszFilename,
240 const char *pszAccess,
241 bool bSetError,
242 CSLConstList /* papszOptions */ ) override;
243
244 int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
245 int nFlags ) override;
246 int Unlink( const char *pszFilename ) override;
247 int Rename( const char *oldpath, const char *newpath ) override;
248 int Mkdir( const char *pszDirname, long nMode ) override;
249 int Rmdir( const char *pszDirname ) override;
250 char **ReadDir( const char *pszDirname ) override
251 { return ReadDirEx(pszDirname, 0); }
252 char **ReadDirEx( const char *pszDirname, int nMaxFiles ) override;
253 char **SiblingFiles( const char *pszFilename ) override;
254
255 int HasOptimizedReadMultiRange( const char* /* pszPath */ )
256 override { return true; }
257
258 const char* GetActualURL(const char* pszFilename) override;
259
260 const char* GetOptions() override;
261
262 char** GetFileMetadata( const char * pszFilename, const char* pszDomain,
263 CSLConstList papszOptions ) override;
264
265 char **ReadDirInternal( const char *pszDirname, int nMaxFiles,
266 bool* pbGotFileList );
267 void InvalidateDirContent( const char *pszDirname );
268
269 virtual const char* GetDebugKey() const { return "VSICURL"; }
270
271 virtual CPLString GetFSPrefix() const { return "/vsicurl/"; }
272 virtual bool AllowCachedDataFor(const char* pszFilename);
273
274 std::shared_ptr<std::string> GetRegion( const char* pszURL,
275 vsi_l_offset nFileOffsetStart );
276
277 void AddRegion( const char* pszURL,
278 vsi_l_offset nFileOffsetStart,
279 size_t nSize,
280 const char *pData );
281
282 bool GetCachedFileProp( const char* pszURL,
283 FileProp& oFileProp );
284 void SetCachedFileProp( const char* pszURL,
285 FileProp& oFileProp );
286 void InvalidateCachedData( const char* pszURL );
287
288 CURLM *GetCurlMultiHandleFor( const CPLString& osURL );
289
290 virtual void ClearCache();
291 virtual void PartialClearCache(const char* pszFilename);
292
293
294 bool GetCachedDirList( const char* pszURL,
295 CachedDirList& oCachedDirList );
296 void SetCachedDirList( const char* pszURL,
297 CachedDirList& oCachedDirList );
298 bool ExistsInCacheDirList( const CPLString& osDirname, bool *pbIsDir );
299
300 virtual CPLString GetURLFromFilename( const CPLString& osFilename );
301 };
302
303 /************************************************************************/
304 /* VSICurlHandle */
305 /************************************************************************/
306
307 class VSICurlHandle : public VSIVirtualHandle
308 {
309 CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
310
311 protected:
312 VSICurlFilesystemHandler* poFS = nullptr;
313
314 bool m_bCached = true;
315
316 FileProp oFileProp{};
317
318 CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
319 char* m_pszURL = nullptr; // e.g "http://example.com/foo"
320 std::string m_osQueryString{}; // e.g. an Azure SAS
321
322 char **m_papszHTTPOptions = nullptr;
323
324 vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
325 int nBlocksToDownload = 1;
326
327 bool bStopOnInterruptUntilUninstall = false;
328 bool bInterrupted = false;
329 VSICurlReadCbkFunc pfnReadCbk = nullptr;
330 void *pReadCbkUserData = nullptr;
331
332 int m_nMaxRetry = 0;
333 double m_dfRetryDelay = 0.0;
334
335 CPLStringList m_aosHeaders{};
336
337 void DownloadRegionPostProcess( const vsi_l_offset startOffset,
338 const int nBlocks,
339 const char* pBuffer,
340 size_t nSize );
341
342 private:
343
344 vsi_l_offset curOffset = 0;
345
346 bool bEOF = false;
347
348 virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
349
350 bool m_bUseHead = false;
351
352 int ReadMultiRangeSingleGet( int nRanges, void ** ppData,
353 const vsi_l_offset* panOffsets,
354 const size_t* panSizes );
355 CPLString GetRedirectURLIfValid(bool& bHasExpired);
356
357 protected:
358 virtual struct curl_slist* GetCurlHeaders( const CPLString& /*osVerb*/,
359 const struct curl_slist* /* psExistingHeaders */)
360 { return nullptr; }
361 virtual bool AllowAutomaticRedirection() { return true; }
362 virtual bool CanRestartOnError( const char*, const char*, bool ) { return false; }
363 virtual bool UseLimitRangeGetInsteadOfHead() { return false; }
364 virtual bool IsDirectoryFromExists( const char* /*pszVerb*/, int /*response_code*/ ) { return false; }
365 virtual void ProcessGetFileSizeResult(const char* /* pszContent */ ) {}
366 void SetURL(const char* pszURL);
367 virtual bool Authenticate() { return false; }
368
369 public:
370
371 VSICurlHandle( VSICurlFilesystemHandler* poFS,
372 const char* pszFilename,
373 const char* pszURLIn = nullptr );
374 ~VSICurlHandle() override;
375
376 int Seek( vsi_l_offset nOffset, int nWhence ) override;
377 vsi_l_offset Tell() override;
378 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
379 int ReadMultiRange( int nRanges, void ** ppData,
380 const vsi_l_offset* panOffsets,
381 const size_t* panSizes ) override;
382 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
383 int Eof() override;
384 int Flush() override;
385 int Close() override;
386
387 bool IsKnownFileSize() const { return oFileProp.bHasComputedFileSize; }
388 vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
389 virtual vsi_l_offset GetFileSize( bool bSetError ) { return GetFileSizeOrHeaders(bSetError, false); }
390 bool Exists( bool bSetError );
391 bool IsDirectory() const { return oFileProp.bIsDirectory; }
392 int GetMode() const { return oFileProp.nMode; }
393 time_t GetMTime() const { return oFileProp.mTime; }
394 const CPLStringList& GetHeaders() { return m_aosHeaders; }
395
396 int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
397 void* pfnUserData,
398 int bStopOnInterruptUntilUninstall );
399 int UninstallReadCbk();
400
401 const char *GetURL() const { return m_pszURL; }
402 };
403
404 /************************************************************************/
405 /* IVSIS3LikeFSHandler */
406 /************************************************************************/
407
408 class IVSIS3LikeFSHandler: public VSICurlFilesystemHandler
409 {
410 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
411
412 bool CopyFile(VSILFILE* fpIn,
413 vsi_l_offset nSourceSize,
414 const char* pszSource,
415 const char* pszTarget,
416 GDALProgressFunc pProgressFunc,
417 void *pProgressData);
418 virtual int MkdirInternal( const char *pszDirname, long nMode, bool bDoStatCheck );
419
420 protected:
421 char** GetFileList( const char *pszFilename,
422 int nMaxFiles,
423 bool* pbGotFileList ) override;
424
425 virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
426 const char* pszURI, bool bAllowNoObject) = 0;
427
428 virtual int CopyObject( const char *oldpath, const char *newpath,
429 CSLConstList papszMetadata );
430
431 IVSIS3LikeFSHandler() = default;
432
433 public:
434 int Unlink( const char *pszFilename ) override;
435 int Mkdir( const char *pszDirname, long nMode ) override;
436 int Rmdir( const char *pszDirname ) override;
437 int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
438 int nFlags ) override;
439 int Rename( const char *oldpath, const char *newpath ) override;
440
441 virtual int DeleteObject( const char *pszFilename );
442
443 virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
444 virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
445
446 virtual CPLString GetStreamingPath( const char* pszFilename ) const;
447
448 bool Sync( const char* pszSource, const char* pszTarget,
449 const char* const * papszOptions,
450 GDALProgressFunc pProgressFunc,
451 void *pProgressData,
452 char*** ppapszOutputs ) override;
453
454 VSIDIR* OpenDir( const char *pszPath, int nRecurseDepth,
455 const char* const *papszOptions) override;
456
457 // Multipart upload
458 virtual bool SupportsParallelMultipartUpload() const { return false; }
459
460 virtual CPLString InitiateMultipartUpload(
461 const std::string& osFilename,
462 IVSIS3LikeHandleHelper *poS3HandleHelper,
463 int nMaxRetry,
464 double dfRetryDelay,
465 CSLConstList papszOptions);
466 virtual CPLString UploadPart(const CPLString& osFilename,
467 int nPartNumber,
468 const std::string& osUploadID,
469 vsi_l_offset nPosition,
470 const void* pabyBuffer,
471 size_t nBufferSize,
472 IVSIS3LikeHandleHelper *poS3HandleHelper,
473 int nMaxRetry,
474 double dfRetryDelay);
475 virtual bool CompleteMultipart(const CPLString& osFilename,
476 const CPLString& osUploadID,
477 const std::vector<CPLString>& aosEtags,
478 vsi_l_offset nTotalSize,
479 IVSIS3LikeHandleHelper *poS3HandleHelper,
480 int nMaxRetry,
481 double dfRetryDelay);
482 virtual bool AbortMultipart(const CPLString& osFilename,
483 const CPLString& osUploadID,
484 IVSIS3LikeHandleHelper *poS3HandleHelper,
485 int nMaxRetry,
486 double dfRetryDelay);
487 };
488
489 /************************************************************************/
490 /* IVSIS3LikeHandle */
491 /************************************************************************/
492
493 class IVSIS3LikeHandle: public VSICurlHandle
494 {
495 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
496
497 protected:
498 bool UseLimitRangeGetInsteadOfHead() override { return true; }
499 bool IsDirectoryFromExists( const char* pszVerb,
500 int response_code ) override
501 {
502 // A bit dirty, but on S3, a GET on a existing directory returns a 416
503 return response_code == 416 && EQUAL(pszVerb, "GET") &&
504 CPLString(m_pszURL).back() == '/';
505 }
506 void ProcessGetFileSizeResult( const char* pszContent ) override
507 {
508 oFileProp.bIsDirectory = strstr(pszContent, "ListBucketResult") != nullptr;
509 }
510
511 public:
512 IVSIS3LikeHandle( VSICurlFilesystemHandler* poFSIn,
513 const char* pszFilename,
514 const char* pszURLIn ) :
515 VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
516 ~IVSIS3LikeHandle() override {}
517 };
518
519 /************************************************************************/
520 /* VSIS3WriteHandle */
521 /************************************************************************/
522
523 class VSIS3WriteHandle final : public VSIVirtualHandle
524 {
525 CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
526
527 IVSIS3LikeFSHandler *m_poFS = nullptr;
528 CPLString m_osFilename{};
529 IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
530 bool m_bUseChunked = false;
531 CPLStringList m_aosOptions{};
532
533 vsi_l_offset m_nCurOffset = 0;
534 int m_nBufferOff = 0;
535 int m_nBufferSize = 0;
536 bool m_bClosed = false;
537 GByte *m_pabyBuffer = nullptr;
538 CPLString m_osUploadID{};
539 int m_nPartNumber = 0;
540 std::vector<CPLString> m_aosEtags{};
541 bool m_bError = false;
542
543 CURLM *m_hCurlMulti = nullptr;
544 CURL *m_hCurl = nullptr;
545 const void *m_pBuffer = nullptr;
546 CPLString m_osCurlErrBuf{};
547 size_t m_nChunkedBufferOff = 0;
548 size_t m_nChunkedBufferSize = 0;
549 size_t m_nWrittenInPUT = 0;
550
551 int m_nMaxRetry = 0;
552 double m_dfRetryDelay = 0.0;
553 WriteFuncStruct m_sWriteFuncHeaderData{};
554
555 bool UploadPart();
556 bool DoSinglePartPUT();
557
558 static size_t ReadCallBackBufferChunked( char *buffer, size_t size,
559 size_t nitems, void *instream );
560 size_t WriteChunked( const void *pBuffer,
561 size_t nSize, size_t nMemb );
562 int FinishChunkedTransfer();
563
564 void InvalidateParentDirectory();
565
566 public:
567 VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
568 const char* pszFilename,
569 IVSIS3LikeHandleHelper* poS3HandleHelper,
570 bool bUseChunked,
571 CSLConstList papszOptions );
572 ~VSIS3WriteHandle() override;
573
574 int Seek( vsi_l_offset nOffset, int nWhence ) override;
575 vsi_l_offset Tell() override;
576 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
577 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
578 int Eof() override;
579 int Close() override;
580
581 bool IsOK() { return m_bUseChunked || m_pabyBuffer != nullptr; }
582 };
583
584 /************************************************************************/
585 /* VSIAppendWriteHandle */
586 /************************************************************************/
587
588 class VSIAppendWriteHandle : public VSIVirtualHandle
589 {
590 CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
591
592 protected:
593
594 VSICurlFilesystemHandler* m_poFS = nullptr;
595 CPLString m_osFSPrefix{};
596 CPLString m_osFilename{};
597
598 vsi_l_offset m_nCurOffset = 0;
599 int m_nBufferOff = 0;
600 int m_nBufferSize = 0;
601 int m_nBufferOffReadCallback = 0;
602 bool m_bClosed = false;
603 GByte *m_pabyBuffer = nullptr;
604 bool m_bError = false;
605
606 static size_t ReadCallBackBuffer( char *buffer, size_t size,
607 size_t nitems, void *instream );
608 virtual bool Send(bool bIsLastBlock) = 0;
609
610 public:
611 VSIAppendWriteHandle( VSICurlFilesystemHandler* poFS,
612 const char* pszFSPrefix,
613 const char* pszFilename,
614 int nChunkSize );
615 virtual ~VSIAppendWriteHandle();
616
617 int Seek( vsi_l_offset nOffset, int nWhence ) override;
618 vsi_l_offset Tell() override;
619 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
620 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
621 int Eof() override;
622 int Close() override;
623
624 bool IsOK() { return m_pabyBuffer != nullptr; }
625 };
626
627 /************************************************************************/
628 /* CurlRequestHelper */
629 /************************************************************************/
630
631 struct CurlRequestHelper
632 {
633 WriteFuncStruct sWriteFuncData{};
634 WriteFuncStruct sWriteFuncHeaderData{};
635 char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
636
637 CurlRequestHelper();
638 ~CurlRequestHelper();
639 long perform(CURL* hCurlHandle,
640 struct curl_slist* headers, // ownership transferred
641 VSICurlFilesystemHandler *poFS,
642 IVSIS3LikeHandleHelper *poS3HandleHelper);
643 };
644
645 /************************************************************************/
646 /* NetworkStatisticsLogger */
647 /************************************************************************/
648
649 class NetworkStatisticsLogger
650 {
651 static int gnEnabled;
652 static NetworkStatisticsLogger gInstance;
653
654 NetworkStatisticsLogger() = default;
655
656 std::mutex m_mutex{};
657
658 struct Counters
659 {
660 GIntBig nHEAD = 0;
661 GIntBig nGET = 0;
662 GIntBig nPUT = 0;
663 GIntBig nPOST = 0;
664 GIntBig nDELETE = 0;
665 GIntBig nGETDownloadedBytes = 0;
666 GIntBig nPUTUploadedBytes = 0;
667 GIntBig nPOSTDownloadedBytes = 0;
668 GIntBig nPOSTUploadedBytes = 0;
669 };
670
671 enum class ContextPathType
672 {
673 FILESYSTEM,
674 FILE,
675 ACTION,
676 };
677
678 struct ContextPathItem
679 {
680 ContextPathType eType;
681 CPLString osName;
682
683 ContextPathItem(ContextPathType eTypeIn, const CPLString& osNameIn):
684 eType(eTypeIn), osName(osNameIn) {}
685
686 bool operator< (const ContextPathItem& other ) const
687 {
688 if( static_cast<int>(eType) < static_cast<int>(other.eType) )
689 return true;
690 if( static_cast<int>(eType) > static_cast<int>(other.eType) )
691 return false;
692 return osName < other.osName;
693 }
694 };
695
696 struct Stats
697 {
698 Counters counters{};
699 std::map<ContextPathItem, Stats> children{};
700
701 void AsJSON(CPLJSONObject& oJSON) const;
702 };
703
704 Stats m_stats{};
705 std::map<GIntBig, std::vector<ContextPathItem>> m_mapThreadIdToContextPath{};
706
707 static void ReadEnabled();
708
709 std::vector<Counters*> GetCountersForContext();
710
711 public:
712
713 static inline bool IsEnabled()
714 {
715 if( gnEnabled < 0)
716 {
717 ReadEnabled();
718 }
719 return gnEnabled == TRUE;
720 }
721
722 static void EnterFileSystem(const char* pszName);
723
724 static void LeaveFileSystem();
725
726 static void EnterFile(const char* pszName);
727
728 static void LeaveFile();
729
730 static void EnterAction(const char* pszName);
731
732 static void LeaveAction();
733
734 static void LogHEAD();
735
736 static void LogGET(size_t nDownloadedBytes);
737
738 static void LogPUT(size_t nUploadedBytes);
739
740 static void LogPOST(size_t nUploadedBytes,
741 size_t nDownloadedBytes);
742
743 static void LogDELETE();
744
745 static void Reset();
746
747 static CPLString GetReportAsSerializedJSON();
748 };
749
750 struct NetworkStatisticsFileSystem
751 {
752 inline explicit NetworkStatisticsFileSystem(const char* pszName) {
753 NetworkStatisticsLogger::EnterFileSystem(pszName);
754 }
755
756 inline ~NetworkStatisticsFileSystem()
757 {
758 NetworkStatisticsLogger::LeaveFileSystem();
759 }
760 };
761
762 struct NetworkStatisticsFile
763 {
764 inline explicit NetworkStatisticsFile(const char* pszName) {
765 NetworkStatisticsLogger::EnterFile(pszName);
766 }
767
768 inline ~NetworkStatisticsFile()
769 {
770 NetworkStatisticsLogger::LeaveFile();
771 }
772 };
773
774 struct NetworkStatisticsAction
775 {
776 inline explicit NetworkStatisticsAction(const char* pszName) {
777 NetworkStatisticsLogger::EnterAction(pszName);
778 }
779
780 inline ~NetworkStatisticsAction()
781 {
782 NetworkStatisticsLogger::LeaveAction();
783 }
784 };
785
786
787 int VSICURLGetDownloadChunkSize();
788
789 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
790 VSILFILE *fp,
791 VSICurlReadCbkFunc pfnReadCbk,
792 void *pReadCbkUserData );
793 size_t VSICurlHandleWriteFunc( void *buffer, size_t count,
794 size_t nmemb, void *req );
795 void MultiPerform(CURLM* hCurlMultiHandle,
796 CURL* hEasyHandle = nullptr);
797 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
798
799 int VSICurlParseUnixPermissions(const char* pszPermissions);
800
801 } // namespace cpl
802
804
805 #endif // HAVE_CURL
806
807 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
CPLJSONObject
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:54
CPLStringList
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:442
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:333
VSIVirtualHandle
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
cpl_json.h
Interface for read and write JSON documents.
cpl_port.h
Core portability definitions for CPL.
EQUAL
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:576
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1007
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1216
GByte
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:215
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:248
cpl_string.h
Various convenience functions for working with strings and string lists.
VSI_L_OFFSET_MAX
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:142
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:319
VSIStatBufL
struct VSI_STAT64_T VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:195
VSILFILE
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:156
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:140