GDAL

cpl_vsil_curl_class.h
1 /******************************************************************************
2 *
3 * Project: CPL - Common Portability Library
4 * Purpose: Declarations for /vsicurl/ and related file systems
5 * Author: Even Rouault, even.rouault at spatialys.com
6 *
7 ******************************************************************************
8 * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the "Software"),
12 * to deal in the Software without restriction, including without limitation
13 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 * and/or sell copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included
18 * in all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 ****************************************************************************/
28
29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
31
32 #ifdef HAVE_CURL
33
34 #include "cpl_aws.h"
35 #include "cpl_azure.h"
36 #include "cpl_port.h"
37 #include "cpl_json.h"
38 #include "cpl_string.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
41
42 #include "cpl_curl_priv.h"
43
44 #include <set>
45 #include <map>
46 #include <memory>
47 #include <mutex>
48
50
51 // Leave it for backward compatibility, but deprecate.
52 #define HAVE_CURLINFO_REDIRECT_URL
53
54 void VSICurlStreamingClearCache( void ); // from cpl_vsil_curl_streaming.cpp
55
56 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL,
57 const char * const* papszOptions);
58 struct curl_slist* VSICurlMergeHeaders( struct curl_slist* poDest,
59 struct curl_slist* poSrcToDestroy );
60
61 struct curl_slist* VSICurlSetContentTypeFromExt(struct curl_slist* polist,
62 const char *pszPath);
63
64 struct curl_slist* VSICurlSetCreationHeadersFromOptions(struct curl_slist* headers,
65 CSLConstList papszOptions,
66 const char *pszPath);
67
68 namespace cpl {
69
70 typedef enum
71 {
72 EXIST_UNKNOWN = -1,
73 EXIST_NO,
74 EXIST_YES,
75 } ExistStatus;
76
77 class FileProp
78 {
79 public:
80 unsigned int nGenerationAuthParameters = 0;
81 ExistStatus eExists = EXIST_UNKNOWN;
82 vsi_l_offset fileSize = 0;
83 time_t mTime = 0;
84 time_t nExpireTimestampLocal = 0;
85 CPLString osRedirectURL{};
86 bool bHasComputedFileSize = false;
87 bool bIsDirectory = false;
88 int nMode = 0; // st_mode member of struct stat
89 bool bS3LikeRedirect = false;
90 CPLString ETag{};
91 };
92
93 struct CachedDirList
94 {
95 bool bGotFileList = false;
96 unsigned int nGenerationAuthParameters = 0;
97 CPLStringList oFileList{}; /* only file name without path */
98 };
99
100 struct WriteFuncStruct
101 {
102 char* pBuffer = nullptr;
103 size_t nSize = 0;
104 bool bIsHTTP = false;
105 bool bMultiRange = false;
106 vsi_l_offset nStartOffset = 0;
107 vsi_l_offset nEndOffset = 0;
108 int nHTTPCode = 0;
109 vsi_l_offset nContentLength = 0;
110 bool bFoundContentRange = false;
111 bool bError = false;
112 bool bInterruptDownload = false;
113 bool bDetectRangeDownloadingError = false;
114 GIntBig nTimestampDate = 0; // Corresponds to Date: header field
115
116 VSILFILE *fp = nullptr;
117 VSICurlReadCbkFunc pfnReadCbk = nullptr;
118 void *pReadCbkUserData = nullptr;
119 bool bInterrupted = false;
120 bool bInterruptIfNonErrorPayload = false;
121
122 #if !CURL_AT_LEAST_VERSION(7,54,0)
123 // Workaround to ignore extra HTTP response headers from
124 // proxies in older versions of curl.
125 // CURLOPT_SUPPRESS_CONNECT_HEADERS fixes this
126 bool bIsProxyConnectHeader = false;
127 #endif
128 };
129
130 struct PutData
131 {
132 const GByte* pabyData = nullptr;
133 size_t nOff = 0;
134 size_t nTotalSize = 0;
135
136 static size_t ReadCallBackBuffer( char *buffer, size_t size,
137 size_t nitems, void *instream )
138 {
139 PutData* poThis = static_cast<PutData *>(instream);
140 const size_t nSizeMax = size * nitems;
141 const size_t nSizeToWrite =
142 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
143 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
144 poThis->nOff += nSizeToWrite;
145 return nSizeToWrite;
146 }
147 };
148
149 /************************************************************************/
150 /* VSICurlFilesystemHandler */
151 /************************************************************************/
152
153 class VSICurlHandle;
154
155 class VSICurlFilesystemHandlerBase : public VSIFilesystemHandler
156 {
157 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBase)
158
159 struct FilenameOffsetPair
160 {
161 std::string filename_;
162 vsi_l_offset offset_;
163
164 FilenameOffsetPair(const std::string& filename,
165 vsi_l_offset offset) :
166 filename_(filename), offset_(offset) {}
167
168 bool operator==(const FilenameOffsetPair& other) const
169 {
170 return filename_ == other.filename_ &&
171 offset_ == other.offset_;
172 }
173 };
174 struct FilenameOffsetPairHasher
175 {
176 std::size_t operator()(const FilenameOffsetPair& k) const
177 {
178 return std::hash<std::string>()(k.filename_) ^
179 std::hash<vsi_l_offset>()(k.offset_);
180 }
181 };
182
183 using RegionCacheType =
184 lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
185 lru11::NullLock,
186 std::unordered_map<
187 FilenameOffsetPair,
188 typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
189 std::shared_ptr<std::string>>>::iterator,
190 FilenameOffsetPairHasher>>;
191
192 std::unique_ptr<RegionCacheType> m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use GetRegionCache();
193 RegionCacheType* GetRegionCache();
194
195 // LRU cache that just keeps in memory if this file system handler is
196 // spposed to know the file properties of a file. The actual cache is a
197 // shared one among all network file systems.
198 // The aim of that design is that invalidating /vsis3/foo results in
199 // /vsis3_streaming/foo to be invalidated as well.
200 lru11::Cache<std::string, bool> oCacheFileProp;
201
202 int nCachedFilesInDirList = 0;
203 lru11::Cache<std::string, CachedDirList> oCacheDirList;
204
205 char** ParseHTMLFileList(const char* pszFilename,
206 int nMaxFiles,
207 char* pszData,
208 bool* pbGotFileList);
209
210 protected:
211 CPLMutex *hMutex = nullptr;
212
213 virtual VSICurlHandle* CreateFileHandle(const char* pszFilename);
214 virtual char** GetFileList(const char *pszFilename,
215 int nMaxFiles,
216 bool* pbGotFileList);
217
218 void RegisterEmptyDir( const CPLString& osDirname );
219
220 bool AnalyseS3FileList( const CPLString& osBaseURL,
221 const char* pszXML,
222 CPLStringList& osFileList,
223 int nMaxFiles,
224 const std::set<std::string>& oSetIgnoredStorageClasses,
225 bool& bIsTruncated );
226
227 void AnalyseSwiftFileList( const CPLString& osBaseURL,
228 const CPLString& osPrefix,
229 const char* pszJson,
230 CPLStringList& osFileList,
231 int nMaxFilesThisQuery,
232 int nMaxFiles,
233 bool& bIsTruncated,
234 CPLString& osNextMarker );
235
236 static const char* GetOptionsStatic();
237
238 static bool IsAllowedFilename( const char* pszFilename );
239
240 VSICurlFilesystemHandlerBase();
241
242 public:
243 ~VSICurlFilesystemHandlerBase() override;
244
245 VSIVirtualHandle *Open( const char *pszFilename,
246 const char *pszAccess,
247 bool bSetError,
248 CSLConstList /* papszOptions */ ) override;
249
250 int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
251 int nFlags ) override;
252 int Unlink( const char *pszFilename ) override;
253 int Rename( const char *oldpath, const char *newpath ) override;
254 int Mkdir( const char *pszDirname, long nMode ) override;
255 int Rmdir( const char *pszDirname ) override;
256 char **ReadDir( const char *pszDirname ) override
257 { return ReadDirEx(pszDirname, 0); }
258 char **ReadDirEx( const char *pszDirname, int nMaxFiles ) override;
259 char **SiblingFiles( const char *pszFilename ) override;
260
261 int HasOptimizedReadMultiRange( const char* /* pszPath */ )
262 override { return true; }
263
264 const char* GetActualURL(const char* pszFilename) override;
265
266 const char* GetOptions() override;
267
268 char** GetFileMetadata( const char * pszFilename, const char* pszDomain,
269 CSLConstList papszOptions ) override;
270
271 char **ReadDirInternal( const char *pszDirname, int nMaxFiles,
272 bool* pbGotFileList );
273 void InvalidateDirContent( const char *pszDirname );
274
275 virtual const char* GetDebugKey() const = 0;
276
277 virtual CPLString GetFSPrefix() const = 0;
278 virtual bool AllowCachedDataFor(const char* pszFilename);
279
280 virtual bool IsLocal( const char* /* pszPath */ ) override { return false; }
281 virtual bool SupportsSequentialWrite( const char* /* pszPath */, bool /* bAllowLocalTempFile */ ) override { return false; }
282 virtual bool SupportsRandomWrite( const char* /* pszPath */, bool /* bAllowLocalTempFile */ ) override { return false; }
283
284 std::shared_ptr<std::string> GetRegion( const char* pszURL,
285 vsi_l_offset nFileOffsetStart );
286
287 void AddRegion( const char* pszURL,
288 vsi_l_offset nFileOffsetStart,
289 size_t nSize,
290 const char *pData );
291
292 bool GetCachedFileProp( const char* pszURL,
293 FileProp& oFileProp );
294 void SetCachedFileProp( const char* pszURL,
295 FileProp& oFileProp );
296 void InvalidateCachedData( const char* pszURL );
297
298 CURLM *GetCurlMultiHandleFor( const CPLString& osURL );
299
300 virtual void ClearCache();
301 virtual void PartialClearCache(const char* pszFilename);
302
303
304 bool GetCachedDirList( const char* pszURL,
305 CachedDirList& oCachedDirList );
306 void SetCachedDirList( const char* pszURL,
307 CachedDirList& oCachedDirList );
308 bool ExistsInCacheDirList( const CPLString& osDirname, bool *pbIsDir );
309
310 virtual CPLString GetURLFromFilename( const CPLString& osFilename );
311
312 std::string GetStreamingFilename(const std::string& osFilename) const override = 0;
313
314 static std::set<std::string> GetS3IgnoredStorageClasses();
315 };
316
317
318 class VSICurlFilesystemHandler: public VSICurlFilesystemHandlerBase
319 {
320 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
321
322 public:
323 VSICurlFilesystemHandler() = default;
324
325 const char* GetDebugKey() const override { return "VSICURL"; }
326
327 CPLString GetFSPrefix() const override { return "/vsicurl/"; }
328
329 std::string GetStreamingFilename(const std::string& osFilename) const override;
330 };
331
332 /************************************************************************/
333 /* VSICurlHandle */
334 /************************************************************************/
335
336 class VSICurlHandle : public VSIVirtualHandle
337 {
338 CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
339
340 protected:
341 VSICurlFilesystemHandlerBase* poFS = nullptr;
342
343 bool m_bCached = true;
344
345 mutable FileProp oFileProp{};
346
347 mutable std::mutex m_oMutex{};
348 CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
349 char* m_pszURL = nullptr; // e.g "http://example.com/foo"
350 mutable std::string m_osQueryString{}; // e.g. an Azure SAS
351
352 char **m_papszHTTPOptions = nullptr;
353
354 vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
355 int nBlocksToDownload = 1;
356
357 bool bStopOnInterruptUntilUninstall = false;
358 bool bInterrupted = false;
359 VSICurlReadCbkFunc pfnReadCbk = nullptr;
360 void *pReadCbkUserData = nullptr;
361
362 int m_nMaxRetry = 0;
363 double m_dfRetryDelay = 0.0;
364
365 CPLStringList m_aosHeaders{};
366
367 void DownloadRegionPostProcess( const vsi_l_offset startOffset,
368 const int nBlocks,
369 const char* pBuffer,
370 size_t nSize );
371
372 private:
373
374 vsi_l_offset curOffset = 0;
375
376 bool bEOF = false;
377
378 virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
379
380 bool m_bUseHead = false;
381 bool m_bUseRedirectURLIfNoQueryStringParams = false;
382
383 // Specific to Planetary Computer signing: https://planetarycomputer.microsoft.com/docs/concepts/sas/
384 mutable bool m_bPlanetaryComputerURLSigning = false;
385 mutable std::string m_osPlanetaryComputerCollection{};
386 void ManagePlanetaryComputerSigning() const;
387
388 int ReadMultiRangeSingleGet( int nRanges, void ** ppData,
389 const vsi_l_offset* panOffsets,
390 const size_t* panSizes );
391 CPLString GetRedirectURLIfValid(bool& bHasExpired) const;
392
393 void UpdateRedirectInfo( CURL* hCurlHandle,
394 const WriteFuncStruct& sWriteFuncHeaderData );
395
396 protected:
397 virtual struct curl_slist* GetCurlHeaders( const CPLString& /*osVerb*/,
398 const struct curl_slist* /* psExistingHeaders */)
399 { return nullptr; }
400 virtual bool AllowAutomaticRedirection() { return true; }
401 virtual bool CanRestartOnError( const char*, const char*, bool ) { return false; }
402 virtual bool UseLimitRangeGetInsteadOfHead() { return false; }
403 virtual bool IsDirectoryFromExists( const char* /*pszVerb*/, int /*response_code*/ ) { return false; }
404 virtual void ProcessGetFileSizeResult(const char* /* pszContent */ ) {}
405 void SetURL(const char* pszURL);
406 virtual bool Authenticate(const char* /* pszFilename */) { return false; }
407
408 public:
409
410 VSICurlHandle( VSICurlFilesystemHandlerBase* poFS,
411 const char* pszFilename,
412 const char* pszURLIn = nullptr );
413 ~VSICurlHandle() override;
414
415 int Seek( vsi_l_offset nOffset, int nWhence ) override;
416 vsi_l_offset Tell() override;
417 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
418 int ReadMultiRange( int nRanges, void ** ppData,
419 const vsi_l_offset* panOffsets,
420 const size_t* panSizes ) override;
421 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
422 int Eof() override;
423 int Flush() override;
424 int Close() override;
425
426 bool HasPRead() const override { return true; }
427 size_t PRead( void* pBuffer, size_t nSize, vsi_l_offset nOffset ) const override;
428
429 bool IsKnownFileSize() const { return oFileProp.bHasComputedFileSize; }
430 vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
431 virtual vsi_l_offset GetFileSize( bool bSetError ) { return GetFileSizeOrHeaders(bSetError, false); }
432 bool Exists( bool bSetError );
433 bool IsDirectory() const { return oFileProp.bIsDirectory; }
434 int GetMode() const { return oFileProp.nMode; }
435 time_t GetMTime() const { return oFileProp.mTime; }
436 const CPLStringList& GetHeaders() { return m_aosHeaders; }
437
438 int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
439 void* pfnUserData,
440 int bStopOnInterruptUntilUninstall );
441 int UninstallReadCbk();
442
443 const char *GetURL() const { return m_pszURL; }
444 };
445
446 /************************************************************************/
447 /* IVSIS3LikeFSHandler */
448 /************************************************************************/
449
450 class IVSIS3LikeFSHandler: public VSICurlFilesystemHandlerBase
451 {
452 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
453
454 bool CopyFile(VSILFILE* fpIn,
455 vsi_l_offset nSourceSize,
456 const char* pszSource,
457 const char* pszTarget,
458 CSLConstList papszOptions,
459 GDALProgressFunc pProgressFunc,
460 void *pProgressData);
461 virtual int MkdirInternal( const char *pszDirname, long nMode, bool bDoStatCheck );
462
463 protected:
464 char** GetFileList( const char *pszFilename,
465 int nMaxFiles,
466 bool* pbGotFileList ) override;
467
468 virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
469 const char* pszURI, bool bAllowNoObject) = 0;
470
471 virtual int CopyObject( const char *oldpath, const char *newpath,
472 CSLConstList papszMetadata );
473
474 int RmdirRecursiveInternal( const char* pszDirname, int nBatchSize);
475
476 virtual bool IsAllowedHeaderForObjectCreation( const char* /* pszHeaderName */ ) { return false; }
477
478 IVSIS3LikeFSHandler() = default;
479
480 public:
481 int Unlink( const char *pszFilename ) override;
482 int Mkdir( const char *pszDirname, long nMode ) override;
483 int Rmdir( const char *pszDirname ) override;
484 int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
485 int nFlags ) override;
486 int Rename( const char *oldpath, const char *newpath ) override;
487
488 virtual int DeleteObject( const char *pszFilename );
489
490 virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
491 virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
492
493 bool Sync( const char* pszSource, const char* pszTarget,
494 const char* const * papszOptions,
495 GDALProgressFunc pProgressFunc,
496 void *pProgressData,
497 char*** ppapszOutputs ) override;
498
499 VSIDIR* OpenDir( const char *pszPath, int nRecurseDepth,
500 const char* const *papszOptions) override;
501
502 // Multipart upload
503 virtual bool SupportsParallelMultipartUpload() const { return false; }
504
505 virtual CPLString InitiateMultipartUpload(
506 const std::string& osFilename,
507 IVSIS3LikeHandleHelper *poS3HandleHelper,
508 int nMaxRetry,
509 double dfRetryDelay,
510 CSLConstList papszOptions);
511 virtual CPLString UploadPart(const CPLString& osFilename,
512 int nPartNumber,
513 const std::string& osUploadID,
514 vsi_l_offset nPosition,
515 const void* pabyBuffer,
516 size_t nBufferSize,
517 IVSIS3LikeHandleHelper *poS3HandleHelper,
518 int nMaxRetry,
519 double dfRetryDelay,
520 CSLConstList papszOptions);
521 virtual bool CompleteMultipart(const CPLString& osFilename,
522 const CPLString& osUploadID,
523 const std::vector<CPLString>& aosEtags,
524 vsi_l_offset nTotalSize,
525 IVSIS3LikeHandleHelper *poS3HandleHelper,
526 int nMaxRetry,
527 double dfRetryDelay);
528 virtual bool AbortMultipart(const CPLString& osFilename,
529 const CPLString& osUploadID,
530 IVSIS3LikeHandleHelper *poS3HandleHelper,
531 int nMaxRetry,
532 double dfRetryDelay);
533
534 bool AbortPendingUploads(const char* pszFilename) override;
535 };
536
537 /************************************************************************/
538 /* IVSIS3LikeHandle */
539 /************************************************************************/
540
541 class IVSIS3LikeHandle: public VSICurlHandle
542 {
543 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
544
545 protected:
546 bool UseLimitRangeGetInsteadOfHead() override { return true; }
547 bool IsDirectoryFromExists( const char* pszVerb,
548 int response_code ) override
549 {
550 // A bit dirty, but on S3, a GET on a existing directory returns a 416
551 return response_code == 416 && EQUAL(pszVerb, "GET") &&
552 CPLString(m_pszURL).back() == '/';
553 }
554 void ProcessGetFileSizeResult( const char* pszContent ) override
555 {
556 oFileProp.bIsDirectory = strstr(pszContent, "ListBucketResult") != nullptr;
557 }
558
559 public:
560 IVSIS3LikeHandle( VSICurlFilesystemHandlerBase* poFSIn,
561 const char* pszFilename,
562 const char* pszURLIn ) :
563 VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
564 ~IVSIS3LikeHandle() override {}
565 };
566
567 /************************************************************************/
568 /* VSIS3WriteHandle */
569 /************************************************************************/
570
571 class VSIS3WriteHandle final : public VSIVirtualHandle
572 {
573 CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
574
575 IVSIS3LikeFSHandler *m_poFS = nullptr;
576 CPLString m_osFilename{};
577 IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
578 bool m_bUseChunked = false;
579 CPLStringList m_aosOptions{};
580
581 vsi_l_offset m_nCurOffset = 0;
582 int m_nBufferOff = 0;
583 int m_nBufferSize = 0;
584 bool m_bClosed = false;
585 GByte *m_pabyBuffer = nullptr;
586 CPLString m_osUploadID{};
587 int m_nPartNumber = 0;
588 std::vector<CPLString> m_aosEtags{};
589 bool m_bError = false;
590
591 CURLM *m_hCurlMulti = nullptr;
592 CURL *m_hCurl = nullptr;
593 const void *m_pBuffer = nullptr;
594 CPLString m_osCurlErrBuf{};
595 size_t m_nChunkedBufferOff = 0;
596 size_t m_nChunkedBufferSize = 0;
597 size_t m_nWrittenInPUT = 0;
598
599 int m_nMaxRetry = 0;
600 double m_dfRetryDelay = 0.0;
601 WriteFuncStruct m_sWriteFuncHeaderData{};
602
603 bool UploadPart();
604 bool DoSinglePartPUT();
605
606 static size_t ReadCallBackBufferChunked( char *buffer, size_t size,
607 size_t nitems, void *instream );
608 size_t WriteChunked( const void *pBuffer,
609 size_t nSize, size_t nMemb );
610 int FinishChunkedTransfer();
611
612 void InvalidateParentDirectory();
613
614 public:
615 VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
616 const char* pszFilename,
617 IVSIS3LikeHandleHelper* poS3HandleHelper,
618 bool bUseChunked,
619 CSLConstList papszOptions );
620 ~VSIS3WriteHandle() override;
621
622 int Seek( vsi_l_offset nOffset, int nWhence ) override;
623 vsi_l_offset Tell() override;
624 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
625 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
626 int Eof() override;
627 int Close() override;
628
629 bool IsOK() { return m_bUseChunked || m_pabyBuffer != nullptr; }
630 };
631
632 /************************************************************************/
633 /* VSIAppendWriteHandle */
634 /************************************************************************/
635
636 class VSIAppendWriteHandle : public VSIVirtualHandle
637 {
638 CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
639
640 protected:
641
642 VSICurlFilesystemHandlerBase* m_poFS = nullptr;
643 CPLString m_osFSPrefix{};
644 CPLString m_osFilename{};
645
646 vsi_l_offset m_nCurOffset = 0;
647 int m_nBufferOff = 0;
648 int m_nBufferSize = 0;
649 int m_nBufferOffReadCallback = 0;
650 bool m_bClosed = false;
651 GByte *m_pabyBuffer = nullptr;
652 bool m_bError = false;
653
654 static size_t ReadCallBackBuffer( char *buffer, size_t size,
655 size_t nitems, void *instream );
656 virtual bool Send(bool bIsLastBlock) = 0;
657
658 public:
659 VSIAppendWriteHandle( VSICurlFilesystemHandlerBase* poFS,
660 const char* pszFSPrefix,
661 const char* pszFilename,
662 int nChunkSize );
663 virtual ~VSIAppendWriteHandle();
664
665 int Seek( vsi_l_offset nOffset, int nWhence ) override;
666 vsi_l_offset Tell() override;
667 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
668 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
669 int Eof() override;
670 int Close() override;
671
672 bool IsOK() { return m_pabyBuffer != nullptr; }
673 };
674
675 /************************************************************************/
676 /* VSIDIRWithMissingDirSynthesis */
677 /************************************************************************/
678
679 struct VSIDIRWithMissingDirSynthesis: public VSIDIR
680 {
681 std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
682
683 protected:
684 std::vector<std::string> m_aosSubpathsStack{};
685
686 void SynthetizeMissingDirectories(const std::string& osCurSubdir,
687 bool bAddEntryForThisSubdir);
688 };
689
690 /************************************************************************/
691 /* CurlRequestHelper */
692 /************************************************************************/
693
694 struct CurlRequestHelper
695 {
696 WriteFuncStruct sWriteFuncData{};
697 WriteFuncStruct sWriteFuncHeaderData{};
698 char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
699
700 CurlRequestHelper();
701 ~CurlRequestHelper();
702 long perform(CURL* hCurlHandle,
703 struct curl_slist* headers, // ownership transferred
704 VSICurlFilesystemHandlerBase *poFS,
705 IVSIS3LikeHandleHelper *poS3HandleHelper);
706 };
707
708 /************************************************************************/
709 /* NetworkStatisticsLogger */
710 /************************************************************************/
711
712 class NetworkStatisticsLogger
713 {
714 static int gnEnabled;
715 static NetworkStatisticsLogger gInstance;
716
717 NetworkStatisticsLogger() = default;
718
719 std::mutex m_mutex{};
720
721 struct Counters
722 {
723 GIntBig nHEAD = 0;
724 GIntBig nGET = 0;
725 GIntBig nPUT = 0;
726 GIntBig nPOST = 0;
727 GIntBig nDELETE = 0;
728 GIntBig nGETDownloadedBytes = 0;
729 GIntBig nPUTUploadedBytes = 0;
730 GIntBig nPOSTDownloadedBytes = 0;
731 GIntBig nPOSTUploadedBytes = 0;
732 };
733
734 enum class ContextPathType
735 {
736 FILESYSTEM,
737 FILE,
738 ACTION,
739 };
740
741 struct ContextPathItem
742 {
743 ContextPathType eType;
744 CPLString osName;
745
746 ContextPathItem(ContextPathType eTypeIn, const CPLString& osNameIn):
747 eType(eTypeIn), osName(osNameIn) {}
748
749 bool operator< (const ContextPathItem& other ) const
750 {
751 if( static_cast<int>(eType) < static_cast<int>(other.eType) )
752 return true;
753 if( static_cast<int>(eType) > static_cast<int>(other.eType) )
754 return false;
755 return osName < other.osName;
756 }
757 };
758
759 struct Stats
760 {
761 Counters counters{};
762 std::map<ContextPathItem, Stats> children{};
763
764 void AsJSON(CPLJSONObject& oJSON) const;
765 };
766
767 // Workaround bug in Coverity Scan
768 // coverity[generated_default_constructor_used_in_field_initializer]
769 Stats m_stats{};
770 std::map<GIntBig, std::vector<ContextPathItem>> m_mapThreadIdToContextPath{};
771
772 static void ReadEnabled();
773
774 std::vector<Counters*> GetCountersForContext();
775
776 public:
777
778 static inline bool IsEnabled()
779 {
780 if( gnEnabled < 0)
781 {
782 ReadEnabled();
783 }
784 return gnEnabled == TRUE;
785 }
786
787 static void EnterFileSystem(const char* pszName);
788
789 static void LeaveFileSystem();
790
791 static void EnterFile(const char* pszName);
792
793 static void LeaveFile();
794
795 static void EnterAction(const char* pszName);
796
797 static void LeaveAction();
798
799 static void LogHEAD();
800
801 static void LogGET(size_t nDownloadedBytes);
802
803 static void LogPUT(size_t nUploadedBytes);
804
805 static void LogPOST(size_t nUploadedBytes,
806 size_t nDownloadedBytes);
807
808 static void LogDELETE();
809
810 static void Reset();
811
812 static CPLString GetReportAsSerializedJSON();
813 };
814
815 struct NetworkStatisticsFileSystem
816 {
817 inline explicit NetworkStatisticsFileSystem(const char* pszName) {
818 NetworkStatisticsLogger::EnterFileSystem(pszName);
819 }
820
821 inline ~NetworkStatisticsFileSystem()
822 {
823 NetworkStatisticsLogger::LeaveFileSystem();
824 }
825 };
826
827 struct NetworkStatisticsFile
828 {
829 inline explicit NetworkStatisticsFile(const char* pszName) {
830 NetworkStatisticsLogger::EnterFile(pszName);
831 }
832
833 inline ~NetworkStatisticsFile()
834 {
835 NetworkStatisticsLogger::LeaveFile();
836 }
837 };
838
839 struct NetworkStatisticsAction
840 {
841 inline explicit NetworkStatisticsAction(const char* pszName) {
842 NetworkStatisticsLogger::EnterAction(pszName);
843 }
844
845 inline ~NetworkStatisticsAction()
846 {
847 NetworkStatisticsLogger::LeaveAction();
848 }
849 };
850
851
852 int VSICURLGetDownloadChunkSize();
853
854 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
855 VSILFILE *fp,
856 VSICurlReadCbkFunc pfnReadCbk,
857 void *pReadCbkUserData );
858 size_t VSICurlHandleWriteFunc( void *buffer, size_t count,
859 size_t nmemb, void *req );
860 void MultiPerform(CURLM* hCurlMultiHandle,
861 CURL* hEasyHandle = nullptr);
862 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
863
864 int VSICurlParseUnixPermissions(const char* pszPermissions);
865
866 // Cache of file properties (size, etc.)
867 bool VSICURLGetCachedFileProp( const char* pszURL,
868 FileProp& oFileProp );
869 void VSICURLSetCachedFileProp( const char* pszURL,
870 FileProp& oFileProp );
871 void VSICURLInvalidateCachedFileProp( const char* pszURL );
872 void VSICURLInvalidateCachedFilePropPrefix( const char* pszURL );
873 void VSICURLDestroyCacheFileProp();
874
875 } // namespace cpl
876
878
879 #endif // HAVE_CURL
880
881 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
GByte
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:203
CPLStringList
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:429
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:320
EQUAL
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:543
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1056
CPLJSONObject
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:53
cpl_string.h
cpl_json.h
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:142
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:230
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:355
VSIStatBufL
#define VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:200
cpl_port.h
VSIVirtualHandle
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
VSI_L_OFFSET_MAX
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:144
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:930
VSILFILE
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:158