GDAL

cpl_vsil_curl_class.h
1 /******************************************************************************
2 *
3 * Project: CPL - Common Portability Library
4 * Purpose: Declarations for /vsicurl/ and related file systems
5 * Author: Even Rouault, even.rouault at spatialys.com
6 *
7 ******************************************************************************
8 * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the "Software"),
12 * to deal in the Software without restriction, including without limitation
13 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 * and/or sell copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included
18 * in all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 ****************************************************************************/
28
29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
31
32 #ifdef HAVE_CURL
33
34 #include "cpl_aws.h"
35 #include "cpl_azure.h"
36 #include "cpl_port.h"
37 #include "cpl_json.h"
38 #include "cpl_string.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
41
42 #include "cpl_curl_priv.h"
43
44 #include <algorithm>
45 #include <condition_variable>
46 #include <set>
47 #include <map>
48 #include <memory>
49 #include <mutex>
50 #include <thread>
51
52 // To avoid aliasing to CopyFile to CopyFileA on Windows
53 #ifdef CopyFile
54 #undef CopyFile
55 #endif
56
58
59 // Leave it for backward compatibility, but deprecate.
60 #define HAVE_CURLINFO_REDIRECT_URL
61
62 void VSICurlStreamingClearCache(void); // from cpl_vsil_curl_streaming.cpp
63
64 struct curl_slist *VSICurlSetOptions(CURL *hCurlHandle, const char *pszURL,
65 const char *const *papszOptions);
66 struct curl_slist *VSICurlMergeHeaders(struct curl_slist *poDest,
67 struct curl_slist *poSrcToDestroy);
68
69 struct curl_slist *VSICurlSetContentTypeFromExt(struct curl_slist *polist,
70 const char *pszPath);
71
72 struct curl_slist *VSICurlSetCreationHeadersFromOptions(
73 struct curl_slist *headers, CSLConstList papszOptions, const char *pszPath);
74
75 namespace cpl
76 {
77
78 typedef enum
79 {
80 EXIST_UNKNOWN = -1,
81 EXIST_NO,
82 EXIST_YES,
83 } ExistStatus;
84
85 class FileProp
86 {
87 public:
88 unsigned int nGenerationAuthParameters = 0;
89 ExistStatus eExists = EXIST_UNKNOWN;
90 vsi_l_offset fileSize = 0;
91 time_t mTime = 0;
92 time_t nExpireTimestampLocal = 0;
93 CPLString osRedirectURL{};
94 bool bHasComputedFileSize = false;
95 bool bIsDirectory = false;
96 int nMode = 0; // st_mode member of struct stat
97 bool bS3LikeRedirect = false;
98 CPLString ETag{};
99 };
100
101 struct CachedDirList
102 {
103 bool bGotFileList = false;
104 unsigned int nGenerationAuthParameters = 0;
105 CPLStringList oFileList{}; /* only file name without path */
106 };
107
108 struct WriteFuncStruct
109 {
110 char *pBuffer = nullptr;
111 size_t nSize = 0;
112 bool bIsHTTP = false;
113 bool bMultiRange = false;
114 vsi_l_offset nStartOffset = 0;
115 vsi_l_offset nEndOffset = 0;
116 int nHTTPCode = 0;
117 vsi_l_offset nContentLength = 0;
118 bool bFoundContentRange = false;
119 bool bError = false;
120 bool bInterruptDownload = false;
121 bool bDetectRangeDownloadingError = false;
122 GIntBig nTimestampDate = 0; // Corresponds to Date: header field
123
124 VSILFILE *fp = nullptr;
125 VSICurlReadCbkFunc pfnReadCbk = nullptr;
126 void *pReadCbkUserData = nullptr;
127 bool bInterrupted = false;
128
129 #if !CURL_AT_LEAST_VERSION(7, 54, 0)
130 // Workaround to ignore extra HTTP response headers from
131 // proxies in older versions of curl.
132 // CURLOPT_SUPPRESS_CONNECT_HEADERS fixes this
133 bool bIsProxyConnectHeader = false;
134 #endif
135 };
136
137 struct PutData
138 {
139 const GByte *pabyData = nullptr;
140 size_t nOff = 0;
141 size_t nTotalSize = 0;
142
143 static size_t ReadCallBackBuffer(char *buffer, size_t size, size_t nitems,
144 void *instream)
145 {
146 PutData *poThis = static_cast<PutData *>(instream);
147 const size_t nSizeMax = size * nitems;
148 const size_t nSizeToWrite =
149 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
150 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
151 poThis->nOff += nSizeToWrite;
152 return nSizeToWrite;
153 }
154 };
155
156 /************************************************************************/
157 /* VSICurlFilesystemHandler */
158 /************************************************************************/
159
160 class VSICurlHandle;
161
162 class VSICurlFilesystemHandlerBase : public VSIFilesystemHandler
163 {
164 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBase)
165
166 struct FilenameOffsetPair
167 {
168 std::string filename_;
169 vsi_l_offset offset_;
170
171 FilenameOffsetPair(const std::string &filename, vsi_l_offset offset)
172 : filename_(filename), offset_(offset)
173 {
174 }
175
176 bool operator==(const FilenameOffsetPair &other) const
177 {
178 return filename_ == other.filename_ && offset_ == other.offset_;
179 }
180 };
181 struct FilenameOffsetPairHasher
182 {
183 std::size_t operator()(const FilenameOffsetPair &k) const
184 {
185 return std::hash<std::string>()(k.filename_) ^
186 std::hash<vsi_l_offset>()(k.offset_);
187 }
188 };
189
190 using RegionCacheType = lru11::Cache<
191 FilenameOffsetPair, std::shared_ptr<std::string>, lru11::NullLock,
192 std::unordered_map<
193 FilenameOffsetPair,
194 typename std::list<lru11::KeyValuePair<
195 FilenameOffsetPair, std::shared_ptr<std::string>>>::iterator,
196 FilenameOffsetPairHasher>>;
197
198 std::unique_ptr<RegionCacheType>
199 m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use
200 // GetRegionCache();
201 RegionCacheType *GetRegionCache();
202
203 // LRU cache that just keeps in memory if this file system handler is
204 // spposed to know the file properties of a file. The actual cache is a
205 // shared one among all network file systems.
206 // The aim of that design is that invalidating /vsis3/foo results in
207 // /vsis3_streaming/foo to be invalidated as well.
208 lru11::Cache<std::string, bool> oCacheFileProp;
209
210 int nCachedFilesInDirList = 0;
211 lru11::Cache<std::string, CachedDirList> oCacheDirList;
212
213 char **ParseHTMLFileList(const char *pszFilename, int nMaxFiles,
214 char *pszData, bool *pbGotFileList);
215
216 protected:
217 CPLMutex *hMutex = nullptr;
218
219 virtual VSICurlHandle *CreateFileHandle(const char *pszFilename);
220 virtual char **GetFileList(const char *pszFilename, int nMaxFiles,
221 bool *pbGotFileList);
222
223 void RegisterEmptyDir(const CPLString &osDirname);
224
225 bool
226 AnalyseS3FileList(const CPLString &osBaseURL, const char *pszXML,
227 CPLStringList &osFileList, int nMaxFiles,
228 const std::set<std::string> &oSetIgnoredStorageClasses,
229 bool &bIsTruncated);
230
231 void AnalyseSwiftFileList(const CPLString &osBaseURL,
232 const CPLString &osPrefix, const char *pszJson,
233 CPLStringList &osFileList, int nMaxFilesThisQuery,
234 int nMaxFiles, bool &bIsTruncated,
235 CPLString &osNextMarker);
236
237 static const char *GetOptionsStatic();
238
239 static bool IsAllowedFilename(const char *pszFilename);
240
241 VSICurlFilesystemHandlerBase();
242
243 public:
244 ~VSICurlFilesystemHandlerBase() override;
245
246 VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess,
247 bool bSetError,
248 CSLConstList /* papszOptions */) override;
249
250 int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
251 int nFlags) override;
252 int Unlink(const char *pszFilename) override;
253 int Rename(const char *oldpath, const char *newpath) override;
254 int Mkdir(const char *pszDirname, long nMode) override;
255 int Rmdir(const char *pszDirname) override;
256 char **ReadDir(const char *pszDirname) override
257 {
258 return ReadDirEx(pszDirname, 0);
259 }
260 char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
261 char **SiblingFiles(const char *pszFilename) override;
262
263 int HasOptimizedReadMultiRange(const char * /* pszPath */) override
264 {
265 return true;
266 }
267
268 const char *GetActualURL(const char *pszFilename) override;
269
270 const char *GetOptions() override;
271
272 char **GetFileMetadata(const char *pszFilename, const char *pszDomain,
273 CSLConstList papszOptions) override;
274
275 char **ReadDirInternal(const char *pszDirname, int nMaxFiles,
276 bool *pbGotFileList);
277 void InvalidateDirContent(const char *pszDirname);
278
279 virtual const char *GetDebugKey() const = 0;
280
281 virtual CPLString GetFSPrefix() const = 0;
282 virtual bool AllowCachedDataFor(const char *pszFilename);
283
284 virtual bool IsLocal(const char * /* pszPath */) override
285 {
286 return false;
287 }
288 virtual bool
289 SupportsSequentialWrite(const char * /* pszPath */,
290 bool /* bAllowLocalTempFile */) override
291 {
292 return false;
293 }
294 virtual bool SupportsRandomWrite(const char * /* pszPath */,
295 bool /* bAllowLocalTempFile */) override
296 {
297 return false;
298 }
299
300 std::shared_ptr<std::string> GetRegion(const char *pszURL,
301 vsi_l_offset nFileOffsetStart);
302
303 void AddRegion(const char *pszURL, vsi_l_offset nFileOffsetStart,
304 size_t nSize, const char *pData);
305
306 bool GetCachedFileProp(const char *pszURL, FileProp &oFileProp);
307 void SetCachedFileProp(const char *pszURL, FileProp &oFileProp);
308 void InvalidateCachedData(const char *pszURL);
309
310 CURLM *GetCurlMultiHandleFor(const CPLString &osURL);
311
312 virtual void ClearCache();
313 virtual void PartialClearCache(const char *pszFilename);
314
315 bool GetCachedDirList(const char *pszURL, CachedDirList &oCachedDirList);
316 void SetCachedDirList(const char *pszURL, CachedDirList &oCachedDirList);
317 bool ExistsInCacheDirList(const CPLString &osDirname, bool *pbIsDir);
318
319 virtual CPLString GetURLFromFilename(const CPLString &osFilename);
320
321 std::string
322 GetStreamingFilename(const std::string &osFilename) const override = 0;
323
324 static std::set<std::string> GetS3IgnoredStorageClasses();
325 };
326
327 class VSICurlFilesystemHandler : public VSICurlFilesystemHandlerBase
328 {
329 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
330
331 public:
332 VSICurlFilesystemHandler() = default;
333
334 const char *GetDebugKey() const override
335 {
336 return "VSICURL";
337 }
338
339 CPLString GetFSPrefix() const override
340 {
341 return "/vsicurl/";
342 }
343
344 std::string
345 GetStreamingFilename(const std::string &osFilename) const override;
346 };
347
348 /************************************************************************/
349 /* VSICurlHandle */
350 /************************************************************************/
351
352 class VSICurlHandle : public VSIVirtualHandle
353 {
354 CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
355
356 protected:
357 VSICurlFilesystemHandlerBase *poFS = nullptr;
358
359 bool m_bCached = true;
360
361 mutable FileProp oFileProp{};
362
363 mutable std::mutex m_oMutex{};
364 CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
365 char *m_pszURL = nullptr; // e.g "http://example.com/foo"
366 mutable std::string m_osQueryString{}; // e.g. an Azure SAS
367
368 char **m_papszHTTPOptions = nullptr;
369
370 vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
371 int nBlocksToDownload = 1;
372
373 bool bStopOnInterruptUntilUninstall = false;
374 bool bInterrupted = false;
375 VSICurlReadCbkFunc pfnReadCbk = nullptr;
376 void *pReadCbkUserData = nullptr;
377
378 int m_nMaxRetry = 0;
379 double m_dfRetryDelay = 0.0;
380
381 CPLStringList m_aosHeaders{};
382
383 void DownloadRegionPostProcess(const vsi_l_offset startOffset,
384 const int nBlocks, const char *pBuffer,
385 size_t nSize);
386
387 private:
388 vsi_l_offset curOffset = 0;
389
390 bool bEOF = false;
391
392 virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
393
394 bool m_bUseHead = false;
395 bool m_bUseRedirectURLIfNoQueryStringParams = false;
396
397 // Specific to Planetary Computer signing:
398 // https://planetarycomputer.microsoft.com/docs/concepts/sas/
399 mutable bool m_bPlanetaryComputerURLSigning = false;
400 mutable std::string m_osPlanetaryComputerCollection{};
401 void ManagePlanetaryComputerSigning() const;
402
403 int ReadMultiRangeSingleGet(int nRanges, void **ppData,
404 const vsi_l_offset *panOffsets,
405 const size_t *panSizes);
406 CPLString GetRedirectURLIfValid(bool &bHasExpired) const;
407
408 void UpdateRedirectInfo(CURL *hCurlHandle,
409 const WriteFuncStruct &sWriteFuncHeaderData);
410
411 // Used by AdviseRead()
412 struct AdviseReadRange
413 {
414 bool bDone = false;
415 std::mutex oMutex{};
416 std::condition_variable oCV{};
417 vsi_l_offset nStartOffset = 0;
418 size_t nSize = 0;
419 std::vector<GByte> abyData{};
420 };
421 std::vector<std::unique_ptr<AdviseReadRange>> m_aoAdviseReadRanges{};
422 std::thread m_oThreadAdviseRead{};
423
424 protected:
425 virtual struct curl_slist *
426 GetCurlHeaders(const CPLString & /*osVerb*/,
427 const struct curl_slist * /* psExistingHeaders */)
428 {
429 return nullptr;
430 }
431 virtual bool AllowAutomaticRedirection()
432 {
433 return true;
434 }
435 virtual bool CanRestartOnError(const char *, const char *, bool)
436 {
437 return false;
438 }
439 virtual bool UseLimitRangeGetInsteadOfHead()
440 {
441 return false;
442 }
443 virtual bool IsDirectoryFromExists(const char * /*pszVerb*/,
444 int /*response_code*/)
445 {
446 return false;
447 }
448 virtual void ProcessGetFileSizeResult(const char * /* pszContent */)
449 {
450 }
451 void SetURL(const char *pszURL);
452 virtual bool Authenticate(const char * /* pszFilename */)
453 {
454 return false;
455 }
456
457 public:
458 VSICurlHandle(VSICurlFilesystemHandlerBase *poFS, const char *pszFilename,
459 const char *pszURLIn = nullptr);
460 ~VSICurlHandle() override;
461
462 int Seek(vsi_l_offset nOffset, int nWhence) override;
463 vsi_l_offset Tell() override;
464 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
465 int ReadMultiRange(int nRanges, void **ppData,
466 const vsi_l_offset *panOffsets,
467 const size_t *panSizes) override;
468 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
469 int Eof() override;
470 int Flush() override;
471 int Close() override;
472
473 bool HasPRead() const override
474 {
475 return true;
476 }
477 size_t PRead(void *pBuffer, size_t nSize,
478 vsi_l_offset nOffset) const override;
479
480 void AdviseRead(int nRanges, const vsi_l_offset *panOffsets,
481 const size_t *panSizes) override;
482
483 bool IsKnownFileSize() const
484 {
485 return oFileProp.bHasComputedFileSize;
486 }
487 vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
488 virtual vsi_l_offset GetFileSize(bool bSetError)
489 {
490 return GetFileSizeOrHeaders(bSetError, false);
491 }
492 bool Exists(bool bSetError);
493 bool IsDirectory() const
494 {
495 return oFileProp.bIsDirectory;
496 }
497 int GetMode() const
498 {
499 return oFileProp.nMode;
500 }
501 time_t GetMTime() const
502 {
503 return oFileProp.mTime;
504 }
505 const CPLStringList &GetHeaders()
506 {
507 return m_aosHeaders;
508 }
509
510 int InstallReadCbk(VSICurlReadCbkFunc pfnReadCbk, void *pfnUserData,
511 int bStopOnInterruptUntilUninstall);
512 int UninstallReadCbk();
513
514 const char *GetURL() const
515 {
516 return m_pszURL;
517 }
518 };
519
520 /************************************************************************/
521 /* IVSIS3LikeFSHandler */
522 /************************************************************************/
523
524 class IVSIS3LikeFSHandler : public VSICurlFilesystemHandlerBase
525 {
526 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
527
528 virtual int MkdirInternal(const char *pszDirname, long nMode,
529 bool bDoStatCheck);
530
531 protected:
532 char **GetFileList(const char *pszFilename, int nMaxFiles,
533 bool *pbGotFileList) override;
534
535 virtual IVSIS3LikeHandleHelper *CreateHandleHelper(const char *pszURI,
536 bool bAllowNoObject) = 0;
537
538 virtual int CopyObject(const char *oldpath, const char *newpath,
539 CSLConstList papszMetadata);
540
541 int RmdirRecursiveInternal(const char *pszDirname, int nBatchSize);
542
543 virtual bool
544 IsAllowedHeaderForObjectCreation(const char * /* pszHeaderName */)
545 {
546 return false;
547 }
548
549 IVSIS3LikeFSHandler() = default;
550
551 public:
552 int Unlink(const char *pszFilename) override;
553 int Mkdir(const char *pszDirname, long nMode) override;
554 int Rmdir(const char *pszDirname) override;
555 int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
556 int nFlags) override;
557 int Rename(const char *oldpath, const char *newpath) override;
558
559 virtual int CopyFile(const char *pszSource, const char *pszTarget,
560 VSILFILE *fpSource, vsi_l_offset nSourceSize,
561 const char *const *papszOptions,
562 GDALProgressFunc pProgressFunc,
563 void *pProgressData) override;
564
565 virtual int DeleteObject(const char *pszFilename);
566
567 virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper *)
568 {
569 }
570 virtual void UpdateHandleFromMap(IVSIS3LikeHandleHelper *)
571 {
572 }
573
574 bool Sync(const char *pszSource, const char *pszTarget,
575 const char *const *papszOptions, GDALProgressFunc pProgressFunc,
576 void *pProgressData, char ***ppapszOutputs) override;
577
578 VSIDIR *OpenDir(const char *pszPath, int nRecurseDepth,
579 const char *const *papszOptions) override;
580
581 // Multipart upload
582 virtual bool SupportsParallelMultipartUpload() const
583 {
584 return false;
585 }
586
587 virtual CPLString InitiateMultipartUpload(
588 const std::string &osFilename, IVSIS3LikeHandleHelper *poS3HandleHelper,
589 int nMaxRetry, double dfRetryDelay, CSLConstList papszOptions);
590 virtual CPLString UploadPart(const CPLString &osFilename, int nPartNumber,
591 const std::string &osUploadID,
592 vsi_l_offset nPosition, const void *pabyBuffer,
593 size_t nBufferSize,
594 IVSIS3LikeHandleHelper *poS3HandleHelper,
595 int nMaxRetry, double dfRetryDelay,
596 CSLConstList papszOptions);
597 virtual bool CompleteMultipart(const CPLString &osFilename,
598 const CPLString &osUploadID,
599 const std::vector<CPLString> &aosEtags,
600 vsi_l_offset nTotalSize,
601 IVSIS3LikeHandleHelper *poS3HandleHelper,
602 int nMaxRetry, double dfRetryDelay);
603 virtual bool AbortMultipart(const CPLString &osFilename,
604 const CPLString &osUploadID,
605 IVSIS3LikeHandleHelper *poS3HandleHelper,
606 int nMaxRetry, double dfRetryDelay);
607
608 bool AbortPendingUploads(const char *pszFilename) override;
609 };
610
611 /************************************************************************/
612 /* IVSIS3LikeHandle */
613 /************************************************************************/
614
615 class IVSIS3LikeHandle : public VSICurlHandle
616 {
617 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
618
619 protected:
620 bool UseLimitRangeGetInsteadOfHead() override
621 {
622 return true;
623 }
624 bool IsDirectoryFromExists(const char *pszVerb, int response_code) override
625 {
626 // A bit dirty, but on S3, a GET on a existing directory returns a 416
627 return response_code == 416 && EQUAL(pszVerb, "GET") &&
628 CPLString(m_pszURL).back() == '/';
629 }
630 void ProcessGetFileSizeResult(const char *pszContent) override
631 {
632 oFileProp.bIsDirectory =
633 strstr(pszContent, "ListBucketResult") != nullptr;
634 }
635
636 public:
637 IVSIS3LikeHandle(VSICurlFilesystemHandlerBase *poFSIn,
638 const char *pszFilename, const char *pszURLIn)
639 : VSICurlHandle(poFSIn, pszFilename, pszURLIn)
640 {
641 }
642 ~IVSIS3LikeHandle() override
643 {
644 }
645 };
646
647 /************************************************************************/
648 /* VSIS3WriteHandle */
649 /************************************************************************/
650
651 class VSIS3WriteHandle final : public VSIVirtualHandle
652 {
653 CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
654
655 IVSIS3LikeFSHandler *m_poFS = nullptr;
656 CPLString m_osFilename{};
657 IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
658 bool m_bUseChunked = false;
659 CPLStringList m_aosOptions{};
660 CPLStringList m_aosHTTPOptions{};
661
662 vsi_l_offset m_nCurOffset = 0;
663 int m_nBufferOff = 0;
664 int m_nBufferSize = 0;
665 bool m_bClosed = false;
666 GByte *m_pabyBuffer = nullptr;
667 CPLString m_osUploadID{};
668 int m_nPartNumber = 0;
669 std::vector<CPLString> m_aosEtags{};
670 bool m_bError = false;
671
672 CURLM *m_hCurlMulti = nullptr;
673 CURL *m_hCurl = nullptr;
674 const void *m_pBuffer = nullptr;
675 CPLString m_osCurlErrBuf{};
676 size_t m_nChunkedBufferOff = 0;
677 size_t m_nChunkedBufferSize = 0;
678 size_t m_nWrittenInPUT = 0;
679
680 int m_nMaxRetry = 0;
681 double m_dfRetryDelay = 0.0;
682 WriteFuncStruct m_sWriteFuncHeaderData{};
683
684 bool UploadPart();
685 bool DoSinglePartPUT();
686
687 static size_t ReadCallBackBufferChunked(char *buffer, size_t size,
688 size_t nitems, void *instream);
689 size_t WriteChunked(const void *pBuffer, size_t nSize, size_t nMemb);
690 int FinishChunkedTransfer();
691
692 void InvalidateParentDirectory();
693
694 public:
695 VSIS3WriteHandle(IVSIS3LikeFSHandler *poFS, const char *pszFilename,
696 IVSIS3LikeHandleHelper *poS3HandleHelper, bool bUseChunked,
697 CSLConstList papszOptions);
698 ~VSIS3WriteHandle() override;
699
700 int Seek(vsi_l_offset nOffset, int nWhence) override;
701 vsi_l_offset Tell() override;
702 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
703 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
704 int Eof() override;
705 int Close() override;
706
707 bool IsOK()
708 {
709 return m_bUseChunked || m_pabyBuffer != nullptr;
710 }
711 };
712
713 /************************************************************************/
714 /* VSIAppendWriteHandle */
715 /************************************************************************/
716
717 class VSIAppendWriteHandle : public VSIVirtualHandle
718 {
719 CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
720
721 protected:
722 VSICurlFilesystemHandlerBase *m_poFS = nullptr;
723 CPLString m_osFSPrefix{};
724 CPLString m_osFilename{};
725
726 vsi_l_offset m_nCurOffset = 0;
727 int m_nBufferOff = 0;
728 int m_nBufferSize = 0;
729 int m_nBufferOffReadCallback = 0;
730 bool m_bClosed = false;
731 GByte *m_pabyBuffer = nullptr;
732 bool m_bError = false;
733
734 static size_t ReadCallBackBuffer(char *buffer, size_t size, size_t nitems,
735 void *instream);
736 virtual bool Send(bool bIsLastBlock) = 0;
737
738 public:
739 VSIAppendWriteHandle(VSICurlFilesystemHandlerBase *poFS,
740 const char *pszFSPrefix, const char *pszFilename,
741 int nChunkSize);
742 virtual ~VSIAppendWriteHandle();
743
744 int Seek(vsi_l_offset nOffset, int nWhence) override;
745 vsi_l_offset Tell() override;
746 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
747 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
748 int Eof() override;
749 int Close() override;
750
751 bool IsOK()
752 {
753 return m_pabyBuffer != nullptr;
754 }
755 };
756
757 /************************************************************************/
758 /* VSIDIRWithMissingDirSynthesis */
759 /************************************************************************/
760
761 struct VSIDIRWithMissingDirSynthesis : public VSIDIR
762 {
763 std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
764
765 protected:
766 std::vector<std::string> m_aosSubpathsStack{};
767
768 void SynthetizeMissingDirectories(const std::string &osCurSubdir,
769 bool bAddEntryForThisSubdir);
770 };
771
772 /************************************************************************/
773 /* CurlRequestHelper */
774 /************************************************************************/
775
776 struct CurlRequestHelper
777 {
778 WriteFuncStruct sWriteFuncData{};
779 WriteFuncStruct sWriteFuncHeaderData{};
780 char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
781
782 CurlRequestHelper();
783 ~CurlRequestHelper();
784 long perform(CURL *hCurlHandle,
785 struct curl_slist *headers, // ownership transferred
786 VSICurlFilesystemHandlerBase *poFS,
787 IVSIS3LikeHandleHelper *poS3HandleHelper);
788 };
789
790 /************************************************************************/
791 /* NetworkStatisticsLogger */
792 /************************************************************************/
793
794 class NetworkStatisticsLogger
795 {
796 static int gnEnabled;
797 static NetworkStatisticsLogger gInstance;
798
799 NetworkStatisticsLogger() = default;
800
801 std::mutex m_mutex{};
802
803 struct Counters
804 {
805 GIntBig nHEAD = 0;
806 GIntBig nGET = 0;
807 GIntBig nPUT = 0;
808 GIntBig nPOST = 0;
809 GIntBig nDELETE = 0;
810 GIntBig nGETDownloadedBytes = 0;
811 GIntBig nPUTUploadedBytes = 0;
812 GIntBig nPOSTDownloadedBytes = 0;
813 GIntBig nPOSTUploadedBytes = 0;
814 };
815
816 enum class ContextPathType
817 {
818 FILESYSTEM,
819 FILE,
820 ACTION,
821 };
822
823 struct ContextPathItem
824 {
825 ContextPathType eType;
826 CPLString osName;
827
828 ContextPathItem(ContextPathType eTypeIn, const CPLString &osNameIn)
829 : eType(eTypeIn), osName(osNameIn)
830 {
831 }
832
833 bool operator<(const ContextPathItem &other) const
834 {
835 if (static_cast<int>(eType) < static_cast<int>(other.eType))
836 return true;
837 if (static_cast<int>(eType) > static_cast<int>(other.eType))
838 return false;
839 return osName < other.osName;
840 }
841 };
842
843 struct Stats
844 {
845 Counters counters{};
846 std::map<ContextPathItem, Stats> children{};
847
848 void AsJSON(CPLJSONObject &oJSON) const;
849 };
850
851 // Workaround bug in Coverity Scan
852 // coverity[generated_default_constructor_used_in_field_initializer]
853 Stats m_stats{};
854 std::map<GIntBig, std::vector<ContextPathItem>>
855 m_mapThreadIdToContextPath{};
856
857 static void ReadEnabled();
858
859 std::vector<Counters *> GetCountersForContext();
860
861 public:
862 static inline bool IsEnabled()
863 {
864 if (gnEnabled < 0)
865 {
866 ReadEnabled();
867 }
868 return gnEnabled == TRUE;
869 }
870
871 static void EnterFileSystem(const char *pszName);
872
873 static void LeaveFileSystem();
874
875 static void EnterFile(const char *pszName);
876
877 static void LeaveFile();
878
879 static void EnterAction(const char *pszName);
880
881 static void LeaveAction();
882
883 static void LogHEAD();
884
885 static void LogGET(size_t nDownloadedBytes);
886
887 static void LogPUT(size_t nUploadedBytes);
888
889 static void LogPOST(size_t nUploadedBytes, size_t nDownloadedBytes);
890
891 static void LogDELETE();
892
893 static void Reset();
894
895 static CPLString GetReportAsSerializedJSON();
896 };
897
898 struct NetworkStatisticsFileSystem
899 {
900 inline explicit NetworkStatisticsFileSystem(const char *pszName)
901 {
902 NetworkStatisticsLogger::EnterFileSystem(pszName);
903 }
904
905 inline ~NetworkStatisticsFileSystem()
906 {
907 NetworkStatisticsLogger::LeaveFileSystem();
908 }
909 };
910
911 struct NetworkStatisticsFile
912 {
913 inline explicit NetworkStatisticsFile(const char *pszName)
914 {
915 NetworkStatisticsLogger::EnterFile(pszName);
916 }
917
918 inline ~NetworkStatisticsFile()
919 {
920 NetworkStatisticsLogger::LeaveFile();
921 }
922 };
923
924 struct NetworkStatisticsAction
925 {
926 inline explicit NetworkStatisticsAction(const char *pszName)
927 {
928 NetworkStatisticsLogger::EnterAction(pszName);
929 }
930
931 inline ~NetworkStatisticsAction()
932 {
933 NetworkStatisticsLogger::LeaveAction();
934 }
935 };
936
937 int VSICURLGetDownloadChunkSize();
938
939 void VSICURLInitWriteFuncStruct(WriteFuncStruct *psStruct, VSILFILE *fp,
940 VSICurlReadCbkFunc pfnReadCbk,
941 void *pReadCbkUserData);
942 size_t VSICurlHandleWriteFunc(void *buffer, size_t count, size_t nmemb,
943 void *req);
944 void MultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle = nullptr);
945 void VSICURLResetHeaderAndWriterFunctions(CURL *hCurlHandle);
946
947 int VSICurlParseUnixPermissions(const char *pszPermissions);
948
949 // Cache of file properties (size, etc.)
950 bool VSICURLGetCachedFileProp(const char *pszURL, FileProp &oFileProp);
951 void VSICURLSetCachedFileProp(const char *pszURL, FileProp &oFileProp);
952 void VSICURLInvalidateCachedFileProp(const char *pszURL);
953 void VSICURLInvalidateCachedFilePropPrefix(const char *pszURL);
954 void VSICURLDestroyCacheFileProp();
955
956 } // namespace cpl
957
959
960 #endif // HAVE_CURL
961
962 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
GByte
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:205
CPLStringList
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:437
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:311
EQUAL
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:571
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1195
CPLJSONObject
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:54
cpl_string.h
VSIVirtualHandle
Virtual file handle.
Definition: cpl_vsi_virtual.h:61
cpl_json.h
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:146
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:235
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:382
VSIStatBufL
#define VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:203
cpl_port.h
VSI_L_OFFSET_MAX
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:148
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1051