GDAL

cpl_vsil_curl_class.h
1 /******************************************************************************
2 *
3 * Project: CPL - Common Portability Library
4 * Purpose: Declarations for /vsicurl/ and related file systems
5 * Author: Even Rouault, even.rouault at spatialys.com
6 *
7 ******************************************************************************
8 * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the "Software"),
12 * to deal in the Software without restriction, including without limitation
13 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 * and/or sell copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included
18 * in all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 ****************************************************************************/
28
29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
31
32 #ifdef HAVE_CURL
33
34 #include "cpl_aws.h"
35 #include "cpl_azure.h"
36 #include "cpl_port.h"
37 #include "cpl_json.h"
38 #include "cpl_string.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
41
42 #include "cpl_curl_priv.h"
43
44 #include <set>
45 #include <map>
46 #include <memory>
47 #include <mutex>
48
50
51 // Leave it for backward compatibility, but deprecate.
52 #define HAVE_CURLINFO_REDIRECT_URL
53
54 void VSICurlStreamingClearCache( void ); // from cpl_vsil_curl_streaming.cpp
55
56 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL,
57 const char * const* papszOptions);
58 struct curl_slist* VSICurlMergeHeaders( struct curl_slist* poDest,
59 struct curl_slist* poSrcToDestroy );
60
61 struct curl_slist* VSICurlSetContentTypeFromExt(struct curl_slist* polist,
62 const char *pszPath);
63
64 struct curl_slist* VSICurlSetCreationHeadersFromOptions(struct curl_slist* headers,
65 CSLConstList papszOptions,
66 const char *pszPath);
67
68 namespace cpl {
69
70 typedef enum
71 {
72 EXIST_UNKNOWN = -1,
73 EXIST_NO,
74 EXIST_YES,
75 } ExistStatus;
76
77 class FileProp
78 {
79 public:
80 unsigned int nGenerationAuthParameters = 0;
81 ExistStatus eExists = EXIST_UNKNOWN;
82 vsi_l_offset fileSize = 0;
83 time_t mTime = 0;
84 time_t nExpireTimestampLocal = 0;
85 CPLString osRedirectURL{};
86 bool bHasComputedFileSize = false;
87 bool bIsDirectory = false;
88 int nMode = 0; // st_mode member of struct stat
89 bool bS3LikeRedirect = false;
90 CPLString ETag{};
91 };
92
93 struct CachedDirList
94 {
95 bool bGotFileList = false;
96 unsigned int nGenerationAuthParameters = 0;
97 CPLStringList oFileList{}; /* only file name without path */
98 };
99
100 struct WriteFuncStruct
101 {
102 char* pBuffer = nullptr;
103 size_t nSize = 0;
104 bool bIsHTTP = false;
105 bool bMultiRange = false;
106 vsi_l_offset nStartOffset = 0;
107 vsi_l_offset nEndOffset = 0;
108 int nHTTPCode = 0;
109 vsi_l_offset nContentLength = 0;
110 bool bFoundContentRange = false;
111 bool bError = false;
112 bool bInterruptDownload = false;
113 bool bDetectRangeDownloadingError = false;
114 GIntBig nTimestampDate = 0; // Corresponds to Date: header field
115
116 VSILFILE *fp = nullptr;
117 VSICurlReadCbkFunc pfnReadCbk = nullptr;
118 void *pReadCbkUserData = nullptr;
119 bool bInterrupted = false;
120
121 #if !CURL_AT_LEAST_VERSION(7,54,0)
122 // Workaround to ignore extra HTTP response headers from
123 // proxies in older versions of curl.
124 // CURLOPT_SUPPRESS_CONNECT_HEADERS fixes this
125 bool bIsProxyConnectHeader = false;
126 #endif
127 };
128
129 struct PutData
130 {
131 const GByte* pabyData = nullptr;
132 size_t nOff = 0;
133 size_t nTotalSize = 0;
134
135 static size_t ReadCallBackBuffer( char *buffer, size_t size,
136 size_t nitems, void *instream )
137 {
138 PutData* poThis = static_cast<PutData *>(instream);
139 const size_t nSizeMax = size * nitems;
140 const size_t nSizeToWrite =
141 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
142 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
143 poThis->nOff += nSizeToWrite;
144 return nSizeToWrite;
145 }
146 };
147
148 /************************************************************************/
149 /* VSICurlFilesystemHandler */
150 /************************************************************************/
151
152 class VSICurlHandle;
153
154 class VSICurlFilesystemHandlerBase : public VSIFilesystemHandler
155 {
156 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBase)
157
158 struct FilenameOffsetPair
159 {
160 std::string filename_;
161 vsi_l_offset offset_;
162
163 FilenameOffsetPair(const std::string& filename,
164 vsi_l_offset offset) :
165 filename_(filename), offset_(offset) {}
166
167 bool operator==(const FilenameOffsetPair& other) const
168 {
169 return filename_ == other.filename_ &&
170 offset_ == other.offset_;
171 }
172 };
173 struct FilenameOffsetPairHasher
174 {
175 std::size_t operator()(const FilenameOffsetPair& k) const
176 {
177 return std::hash<std::string>()(k.filename_) ^
178 std::hash<vsi_l_offset>()(k.offset_);
179 }
180 };
181
182 using RegionCacheType =
183 lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
184 lru11::NullLock,
185 std::unordered_map<
186 FilenameOffsetPair,
187 typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
188 std::shared_ptr<std::string>>>::iterator,
189 FilenameOffsetPairHasher>>;
190
191 std::unique_ptr<RegionCacheType> m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use GetRegionCache();
192 RegionCacheType* GetRegionCache();
193
194 // LRU cache that just keeps in memory if this file system handler is
195 // spposed to know the file properties of a file. The actual cache is a
196 // shared one among all network file systems.
197 // The aim of that design is that invalidating /vsis3/foo results in
198 // /vsis3_streaming/foo to be invalidated as well.
199 lru11::Cache<std::string, bool> oCacheFileProp;
200
201 int nCachedFilesInDirList = 0;
202 lru11::Cache<std::string, CachedDirList> oCacheDirList;
203
204 char** ParseHTMLFileList(const char* pszFilename,
205 int nMaxFiles,
206 char* pszData,
207 bool* pbGotFileList);
208
209 protected:
210 CPLMutex *hMutex = nullptr;
211
212 virtual VSICurlHandle* CreateFileHandle(const char* pszFilename);
213 virtual char** GetFileList(const char *pszFilename,
214 int nMaxFiles,
215 bool* pbGotFileList);
216
217 void RegisterEmptyDir( const CPLString& osDirname );
218
219 bool AnalyseS3FileList( const CPLString& osBaseURL,
220 const char* pszXML,
221 CPLStringList& osFileList,
222 int nMaxFiles,
223 const std::set<std::string>& oSetIgnoredStorageClasses,
224 bool& bIsTruncated );
225
226 void AnalyseSwiftFileList( const CPLString& osBaseURL,
227 const CPLString& osPrefix,
228 const char* pszJson,
229 CPLStringList& osFileList,
230 int nMaxFilesThisQuery,
231 int nMaxFiles,
232 bool& bIsTruncated,
233 CPLString& osNextMarker );
234
235 static const char* GetOptionsStatic();
236
237 static bool IsAllowedFilename( const char* pszFilename );
238
239 VSICurlFilesystemHandlerBase();
240
241 public:
242 ~VSICurlFilesystemHandlerBase() override;
243
244 VSIVirtualHandle *Open( const char *pszFilename,
245 const char *pszAccess,
246 bool bSetError,
247 CSLConstList /* papszOptions */ ) override;
248
249 int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
250 int nFlags ) override;
251 int Unlink( const char *pszFilename ) override;
252 int Rename( const char *oldpath, const char *newpath ) override;
253 int Mkdir( const char *pszDirname, long nMode ) override;
254 int Rmdir( const char *pszDirname ) override;
255 char **ReadDir( const char *pszDirname ) override
256 { return ReadDirEx(pszDirname, 0); }
257 char **ReadDirEx( const char *pszDirname, int nMaxFiles ) override;
258 char **SiblingFiles( const char *pszFilename ) override;
259
260 int HasOptimizedReadMultiRange( const char* /* pszPath */ )
261 override { return true; }
262
263 const char* GetActualURL(const char* pszFilename) override;
264
265 const char* GetOptions() override;
266
267 char** GetFileMetadata( const char * pszFilename, const char* pszDomain,
268 CSLConstList papszOptions ) override;
269
270 char **ReadDirInternal( const char *pszDirname, int nMaxFiles,
271 bool* pbGotFileList );
272 void InvalidateDirContent( const char *pszDirname );
273
274 virtual const char* GetDebugKey() const = 0;
275
276 virtual CPLString GetFSPrefix() const = 0;
277 virtual bool AllowCachedDataFor(const char* pszFilename);
278
279 std::shared_ptr<std::string> GetRegion( const char* pszURL,
280 vsi_l_offset nFileOffsetStart );
281
282 void AddRegion( const char* pszURL,
283 vsi_l_offset nFileOffsetStart,
284 size_t nSize,
285 const char *pData );
286
287 bool GetCachedFileProp( const char* pszURL,
288 FileProp& oFileProp );
289 void SetCachedFileProp( const char* pszURL,
290 FileProp& oFileProp );
291 void InvalidateCachedData( const char* pszURL );
292
293 CURLM *GetCurlMultiHandleFor( const CPLString& osURL );
294
295 virtual void ClearCache();
296 virtual void PartialClearCache(const char* pszFilename);
297
298
299 bool GetCachedDirList( const char* pszURL,
300 CachedDirList& oCachedDirList );
301 void SetCachedDirList( const char* pszURL,
302 CachedDirList& oCachedDirList );
303 bool ExistsInCacheDirList( const CPLString& osDirname, bool *pbIsDir );
304
305 virtual CPLString GetURLFromFilename( const CPLString& osFilename );
306
307 std::string GetStreamingFilename(const std::string& osFilename) const override = 0;
308
309 static std::set<std::string> GetS3IgnoredStorageClasses();
310 };
311
312
313 class VSICurlFilesystemHandler: public VSICurlFilesystemHandlerBase
314 {
315 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
316
317 public:
318 VSICurlFilesystemHandler() = default;
319
320 const char* GetDebugKey() const override { return "VSICURL"; }
321
322 CPLString GetFSPrefix() const override { return "/vsicurl/"; }
323
324 std::string GetStreamingFilename(const std::string& osFilename) const override;
325 };
326
327 /************************************************************************/
328 /* VSICurlHandle */
329 /************************************************************************/
330
331 class VSICurlHandle : public VSIVirtualHandle
332 {
333 CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
334
335 protected:
336 VSICurlFilesystemHandlerBase* poFS = nullptr;
337
338 bool m_bCached = true;
339
340 FileProp oFileProp{};
341
342 CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
343 char* m_pszURL = nullptr; // e.g "http://example.com/foo"
344 std::string m_osQueryString{}; // e.g. an Azure SAS
345
346 char **m_papszHTTPOptions = nullptr;
347
348 vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
349 int nBlocksToDownload = 1;
350
351 bool bStopOnInterruptUntilUninstall = false;
352 bool bInterrupted = false;
353 VSICurlReadCbkFunc pfnReadCbk = nullptr;
354 void *pReadCbkUserData = nullptr;
355
356 int m_nMaxRetry = 0;
357 double m_dfRetryDelay = 0.0;
358
359 CPLStringList m_aosHeaders{};
360
361 void DownloadRegionPostProcess( const vsi_l_offset startOffset,
362 const int nBlocks,
363 const char* pBuffer,
364 size_t nSize );
365
366 private:
367
368 vsi_l_offset curOffset = 0;
369
370 bool bEOF = false;
371
372 virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
373
374 bool m_bUseHead = false;
375 bool m_bUseRedirectURLIfNoQueryStringParams = false;
376
377 int ReadMultiRangeSingleGet( int nRanges, void ** ppData,
378 const vsi_l_offset* panOffsets,
379 const size_t* panSizes );
380 CPLString GetRedirectURLIfValid(bool& bHasExpired);
381
382 protected:
383 virtual struct curl_slist* GetCurlHeaders( const CPLString& /*osVerb*/,
384 const struct curl_slist* /* psExistingHeaders */)
385 { return nullptr; }
386 virtual bool AllowAutomaticRedirection() { return true; }
387 virtual bool CanRestartOnError( const char*, const char*, bool ) { return false; }
388 virtual bool UseLimitRangeGetInsteadOfHead() { return false; }
389 virtual bool IsDirectoryFromExists( const char* /*pszVerb*/, int /*response_code*/ ) { return false; }
390 virtual void ProcessGetFileSizeResult(const char* /* pszContent */ ) {}
391 void SetURL(const char* pszURL);
392 virtual bool Authenticate(const char* /* pszFilename */) { return false; }
393
394 public:
395
396 VSICurlHandle( VSICurlFilesystemHandlerBase* poFS,
397 const char* pszFilename,
398 const char* pszURLIn = nullptr );
399 ~VSICurlHandle() override;
400
401 int Seek( vsi_l_offset nOffset, int nWhence ) override;
402 vsi_l_offset Tell() override;
403 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
404 int ReadMultiRange( int nRanges, void ** ppData,
405 const vsi_l_offset* panOffsets,
406 const size_t* panSizes ) override;
407 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
408 int Eof() override;
409 int Flush() override;
410 int Close() override;
411
412 bool IsKnownFileSize() const { return oFileProp.bHasComputedFileSize; }
413 vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
414 virtual vsi_l_offset GetFileSize( bool bSetError ) { return GetFileSizeOrHeaders(bSetError, false); }
415 bool Exists( bool bSetError );
416 bool IsDirectory() const { return oFileProp.bIsDirectory; }
417 int GetMode() const { return oFileProp.nMode; }
418 time_t GetMTime() const { return oFileProp.mTime; }
419 const CPLStringList& GetHeaders() { return m_aosHeaders; }
420
421 int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
422 void* pfnUserData,
423 int bStopOnInterruptUntilUninstall );
424 int UninstallReadCbk();
425
426 const char *GetURL() const { return m_pszURL; }
427 };
428
429 /************************************************************************/
430 /* IVSIS3LikeFSHandler */
431 /************************************************************************/
432
433 class IVSIS3LikeFSHandler: public VSICurlFilesystemHandlerBase
434 {
435 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
436
437 bool CopyFile(VSILFILE* fpIn,
438 vsi_l_offset nSourceSize,
439 const char* pszSource,
440 const char* pszTarget,
441 CSLConstList papszOptions,
442 GDALProgressFunc pProgressFunc,
443 void *pProgressData);
444 virtual int MkdirInternal( const char *pszDirname, long nMode, bool bDoStatCheck );
445
446 protected:
447 char** GetFileList( const char *pszFilename,
448 int nMaxFiles,
449 bool* pbGotFileList ) override;
450
451 virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
452 const char* pszURI, bool bAllowNoObject) = 0;
453
454 virtual int CopyObject( const char *oldpath, const char *newpath,
455 CSLConstList papszMetadata );
456
457 int RmdirRecursiveInternal( const char* pszDirname, int nBatchSize);
458
459 virtual bool IsAllowedHeaderForObjectCreation( const char* /* pszHeaderName */ ) { return false; }
460
461 IVSIS3LikeFSHandler() = default;
462
463 public:
464 int Unlink( const char *pszFilename ) override;
465 int Mkdir( const char *pszDirname, long nMode ) override;
466 int Rmdir( const char *pszDirname ) override;
467 int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
468 int nFlags ) override;
469 int Rename( const char *oldpath, const char *newpath ) override;
470
471 virtual int DeleteObject( const char *pszFilename );
472
473 virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
474 virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
475
476 bool Sync( const char* pszSource, const char* pszTarget,
477 const char* const * papszOptions,
478 GDALProgressFunc pProgressFunc,
479 void *pProgressData,
480 char*** ppapszOutputs ) override;
481
482 VSIDIR* OpenDir( const char *pszPath, int nRecurseDepth,
483 const char* const *papszOptions) override;
484
485 // Multipart upload
486 virtual bool SupportsParallelMultipartUpload() const { return false; }
487
488 virtual CPLString InitiateMultipartUpload(
489 const std::string& osFilename,
490 IVSIS3LikeHandleHelper *poS3HandleHelper,
491 int nMaxRetry,
492 double dfRetryDelay,
493 CSLConstList papszOptions);
494 virtual CPLString UploadPart(const CPLString& osFilename,
495 int nPartNumber,
496 const std::string& osUploadID,
497 vsi_l_offset nPosition,
498 const void* pabyBuffer,
499 size_t nBufferSize,
500 IVSIS3LikeHandleHelper *poS3HandleHelper,
501 int nMaxRetry,
502 double dfRetryDelay,
503 CSLConstList papszOptions);
504 virtual bool CompleteMultipart(const CPLString& osFilename,
505 const CPLString& osUploadID,
506 const std::vector<CPLString>& aosEtags,
507 vsi_l_offset nTotalSize,
508 IVSIS3LikeHandleHelper *poS3HandleHelper,
509 int nMaxRetry,
510 double dfRetryDelay);
511 virtual bool AbortMultipart(const CPLString& osFilename,
512 const CPLString& osUploadID,
513 IVSIS3LikeHandleHelper *poS3HandleHelper,
514 int nMaxRetry,
515 double dfRetryDelay);
516
517 bool AbortPendingUploads(const char* pszFilename) override;
518 };
519
520 /************************************************************************/
521 /* IVSIS3LikeHandle */
522 /************************************************************************/
523
524 class IVSIS3LikeHandle: public VSICurlHandle
525 {
526 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
527
528 protected:
529 bool UseLimitRangeGetInsteadOfHead() override { return true; }
530 bool IsDirectoryFromExists( const char* pszVerb,
531 int response_code ) override
532 {
533 // A bit dirty, but on S3, a GET on a existing directory returns a 416
534 return response_code == 416 && EQUAL(pszVerb, "GET") &&
535 CPLString(m_pszURL).back() == '/';
536 }
537 void ProcessGetFileSizeResult( const char* pszContent ) override
538 {
539 oFileProp.bIsDirectory = strstr(pszContent, "ListBucketResult") != nullptr;
540 }
541
542 public:
543 IVSIS3LikeHandle( VSICurlFilesystemHandlerBase* poFSIn,
544 const char* pszFilename,
545 const char* pszURLIn ) :
546 VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
547 ~IVSIS3LikeHandle() override {}
548 };
549
550 /************************************************************************/
551 /* VSIS3WriteHandle */
552 /************************************************************************/
553
554 class VSIS3WriteHandle final : public VSIVirtualHandle
555 {
556 CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
557
558 IVSIS3LikeFSHandler *m_poFS = nullptr;
559 CPLString m_osFilename{};
560 IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
561 bool m_bUseChunked = false;
562 CPLStringList m_aosOptions{};
563
564 vsi_l_offset m_nCurOffset = 0;
565 int m_nBufferOff = 0;
566 int m_nBufferSize = 0;
567 bool m_bClosed = false;
568 GByte *m_pabyBuffer = nullptr;
569 CPLString m_osUploadID{};
570 int m_nPartNumber = 0;
571 std::vector<CPLString> m_aosEtags{};
572 bool m_bError = false;
573
574 CURLM *m_hCurlMulti = nullptr;
575 CURL *m_hCurl = nullptr;
576 const void *m_pBuffer = nullptr;
577 CPLString m_osCurlErrBuf{};
578 size_t m_nChunkedBufferOff = 0;
579 size_t m_nChunkedBufferSize = 0;
580 size_t m_nWrittenInPUT = 0;
581
582 int m_nMaxRetry = 0;
583 double m_dfRetryDelay = 0.0;
584 WriteFuncStruct m_sWriteFuncHeaderData{};
585
586 bool UploadPart();
587 bool DoSinglePartPUT();
588
589 static size_t ReadCallBackBufferChunked( char *buffer, size_t size,
590 size_t nitems, void *instream );
591 size_t WriteChunked( const void *pBuffer,
592 size_t nSize, size_t nMemb );
593 int FinishChunkedTransfer();
594
595 void InvalidateParentDirectory();
596
597 public:
598 VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
599 const char* pszFilename,
600 IVSIS3LikeHandleHelper* poS3HandleHelper,
601 bool bUseChunked,
602 CSLConstList papszOptions );
603 ~VSIS3WriteHandle() override;
604
605 int Seek( vsi_l_offset nOffset, int nWhence ) override;
606 vsi_l_offset Tell() override;
607 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
608 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
609 int Eof() override;
610 int Close() override;
611
612 bool IsOK() { return m_bUseChunked || m_pabyBuffer != nullptr; }
613 };
614
615 /************************************************************************/
616 /* VSIAppendWriteHandle */
617 /************************************************************************/
618
619 class VSIAppendWriteHandle : public VSIVirtualHandle
620 {
621 CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
622
623 protected:
624
625 VSICurlFilesystemHandlerBase* m_poFS = nullptr;
626 CPLString m_osFSPrefix{};
627 CPLString m_osFilename{};
628
629 vsi_l_offset m_nCurOffset = 0;
630 int m_nBufferOff = 0;
631 int m_nBufferSize = 0;
632 int m_nBufferOffReadCallback = 0;
633 bool m_bClosed = false;
634 GByte *m_pabyBuffer = nullptr;
635 bool m_bError = false;
636
637 static size_t ReadCallBackBuffer( char *buffer, size_t size,
638 size_t nitems, void *instream );
639 virtual bool Send(bool bIsLastBlock) = 0;
640
641 public:
642 VSIAppendWriteHandle( VSICurlFilesystemHandlerBase* poFS,
643 const char* pszFSPrefix,
644 const char* pszFilename,
645 int nChunkSize );
646 virtual ~VSIAppendWriteHandle();
647
648 int Seek( vsi_l_offset nOffset, int nWhence ) override;
649 vsi_l_offset Tell() override;
650 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
651 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
652 int Eof() override;
653 int Close() override;
654
655 bool IsOK() { return m_pabyBuffer != nullptr; }
656 };
657
658 /************************************************************************/
659 /* VSIDIRWithMissingDirSynthesis */
660 /************************************************************************/
661
662 struct VSIDIRWithMissingDirSynthesis: public VSIDIR
663 {
664 std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
665
666 protected:
667 std::vector<std::string> m_aosSubpathsStack{};
668
669 void SynthetizeMissingDirectories(const std::string& osCurSubdir,
670 bool bAddEntryForThisSubdir);
671 };
672
673 /************************************************************************/
674 /* CurlRequestHelper */
675 /************************************************************************/
676
677 struct CurlRequestHelper
678 {
679 WriteFuncStruct sWriteFuncData{};
680 WriteFuncStruct sWriteFuncHeaderData{};
681 char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
682
683 CurlRequestHelper();
684 ~CurlRequestHelper();
685 long perform(CURL* hCurlHandle,
686 struct curl_slist* headers, // ownership transferred
687 VSICurlFilesystemHandlerBase *poFS,
688 IVSIS3LikeHandleHelper *poS3HandleHelper);
689 };
690
691 /************************************************************************/
692 /* NetworkStatisticsLogger */
693 /************************************************************************/
694
695 class NetworkStatisticsLogger
696 {
697 static int gnEnabled;
698 static NetworkStatisticsLogger gInstance;
699
700 NetworkStatisticsLogger() = default;
701
702 std::mutex m_mutex{};
703
704 struct Counters
705 {
706 GIntBig nHEAD = 0;
707 GIntBig nGET = 0;
708 GIntBig nPUT = 0;
709 GIntBig nPOST = 0;
710 GIntBig nDELETE = 0;
711 GIntBig nGETDownloadedBytes = 0;
712 GIntBig nPUTUploadedBytes = 0;
713 GIntBig nPOSTDownloadedBytes = 0;
714 GIntBig nPOSTUploadedBytes = 0;
715 };
716
717 enum class ContextPathType
718 {
719 FILESYSTEM,
720 FILE,
721 ACTION,
722 };
723
724 struct ContextPathItem
725 {
726 ContextPathType eType;
727 CPLString osName;
728
729 ContextPathItem(ContextPathType eTypeIn, const CPLString& osNameIn):
730 eType(eTypeIn), osName(osNameIn) {}
731
732 bool operator< (const ContextPathItem& other ) const
733 {
734 if( static_cast<int>(eType) < static_cast<int>(other.eType) )
735 return true;
736 if( static_cast<int>(eType) > static_cast<int>(other.eType) )
737 return false;
738 return osName < other.osName;
739 }
740 };
741
742 struct Stats
743 {
744 Counters counters{};
745 std::map<ContextPathItem, Stats> children{};
746
747 void AsJSON(CPLJSONObject& oJSON) const;
748 };
749
750 // Workaround bug in Coverity Scan
751 // coverity[generated_default_constructor_used_in_field_initializer]
752 Stats m_stats{};
753 std::map<GIntBig, std::vector<ContextPathItem>> m_mapThreadIdToContextPath{};
754
755 static void ReadEnabled();
756
757 std::vector<Counters*> GetCountersForContext();
758
759 public:
760
761 static inline bool IsEnabled()
762 {
763 if( gnEnabled < 0)
764 {
765 ReadEnabled();
766 }
767 return gnEnabled == TRUE;
768 }
769
770 static void EnterFileSystem(const char* pszName);
771
772 static void LeaveFileSystem();
773
774 static void EnterFile(const char* pszName);
775
776 static void LeaveFile();
777
778 static void EnterAction(const char* pszName);
779
780 static void LeaveAction();
781
782 static void LogHEAD();
783
784 static void LogGET(size_t nDownloadedBytes);
785
786 static void LogPUT(size_t nUploadedBytes);
787
788 static void LogPOST(size_t nUploadedBytes,
789 size_t nDownloadedBytes);
790
791 static void LogDELETE();
792
793 static void Reset();
794
795 static CPLString GetReportAsSerializedJSON();
796 };
797
798 struct NetworkStatisticsFileSystem
799 {
800 inline explicit NetworkStatisticsFileSystem(const char* pszName) {
801 NetworkStatisticsLogger::EnterFileSystem(pszName);
802 }
803
804 inline ~NetworkStatisticsFileSystem()
805 {
806 NetworkStatisticsLogger::LeaveFileSystem();
807 }
808 };
809
810 struct NetworkStatisticsFile
811 {
812 inline explicit NetworkStatisticsFile(const char* pszName) {
813 NetworkStatisticsLogger::EnterFile(pszName);
814 }
815
816 inline ~NetworkStatisticsFile()
817 {
818 NetworkStatisticsLogger::LeaveFile();
819 }
820 };
821
822 struct NetworkStatisticsAction
823 {
824 inline explicit NetworkStatisticsAction(const char* pszName) {
825 NetworkStatisticsLogger::EnterAction(pszName);
826 }
827
828 inline ~NetworkStatisticsAction()
829 {
830 NetworkStatisticsLogger::LeaveAction();
831 }
832 };
833
834
835 int VSICURLGetDownloadChunkSize();
836
837 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
838 VSILFILE *fp,
839 VSICurlReadCbkFunc pfnReadCbk,
840 void *pReadCbkUserData );
841 size_t VSICurlHandleWriteFunc( void *buffer, size_t count,
842 size_t nmemb, void *req );
843 void MultiPerform(CURLM* hCurlMultiHandle,
844 CURL* hEasyHandle = nullptr);
845 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
846
847 int VSICurlParseUnixPermissions(const char* pszPermissions);
848
849 // Cache of file properties (size, etc.)
850 bool VSICURLGetCachedFileProp( const char* pszURL,
851 FileProp& oFileProp );
852 void VSICURLSetCachedFileProp( const char* pszURL,
853 FileProp& oFileProp );
854 void VSICURLInvalidateCachedFileProp( const char* pszURL );
855 void VSICURLInvalidateCachedFilePropPrefix( const char* pszURL );
856 void VSICURLDestroyCacheFileProp();
857
858 } // namespace cpl
859
861
862 #endif // HAVE_CURL
863
864 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
GByte
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:203
CPLStringList
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:428
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:319
EQUAL
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:540
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1053
CPLJSONObject
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:53
cpl_string.h
cpl_json.h
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:140
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:230
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:329
VSIStatBufL
#define VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:198
cpl_port.h
VSIVirtualHandle
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
VSI_L_OFFSET_MAX
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:142
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:927
VSILFILE
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:156