GDAL

cpl_vsil_curl_class.h
1 /******************************************************************************
2 *
3 * Project: CPL - Common Portability Library
4 * Purpose: Declarations for /vsicurl/ and related file systems
5 * Author: Even Rouault, even.rouault at spatialys.com
6 *
7 ******************************************************************************
8 * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the "Software"),
12 * to deal in the Software without restriction, including without limitation
13 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 * and/or sell copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included
18 * in all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 ****************************************************************************/
28
29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
31
32 #ifdef HAVE_CURL
33
34 #include "cpl_aws.h"
35 #include "cpl_azure.h"
36 #include "cpl_port.h"
37 #include "cpl_json.h"
38 #include "cpl_string.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
41
42 #include "cpl_curl_priv.h"
43
44 #include <algorithm>
45 #include <condition_variable>
46 #include <set>
47 #include <map>
48 #include <memory>
49 #include <mutex>
50 #include <thread>
51
52 // To avoid aliasing to CopyFile to CopyFileA on Windows
53 #ifdef CopyFile
54 #undef CopyFile
55 #endif
56
58
59 // Leave it for backward compatibility, but deprecate.
60 #define HAVE_CURLINFO_REDIRECT_URL
61
62 void VSICurlStreamingClearCache(void); // from cpl_vsil_curl_streaming.cpp
63
64 struct curl_slist *VSICurlSetOptions(CURL *hCurlHandle, const char *pszURL,
65 const char *const *papszOptions);
66 struct curl_slist *VSICurlMergeHeaders(struct curl_slist *poDest,
67 struct curl_slist *poSrcToDestroy);
68
69 struct curl_slist *VSICurlSetContentTypeFromExt(struct curl_slist *polist,
70 const char *pszPath);
71
72 struct curl_slist *VSICurlSetCreationHeadersFromOptions(
73 struct curl_slist *headers, CSLConstList papszOptions, const char *pszPath);
74
75 namespace cpl
76 {
77
78 typedef enum
79 {
80 EXIST_UNKNOWN = -1,
81 EXIST_NO,
82 EXIST_YES,
83 } ExistStatus;
84
85 class FileProp
86 {
87 public:
88 unsigned int nGenerationAuthParameters = 0;
89 ExistStatus eExists = EXIST_UNKNOWN;
90 vsi_l_offset fileSize = 0;
91 time_t mTime = 0;
92 time_t nExpireTimestampLocal = 0;
93 CPLString osRedirectURL{};
94 bool bHasComputedFileSize = false;
95 bool bIsDirectory = false;
96 int nMode = 0; // st_mode member of struct stat
97 bool bS3LikeRedirect = false;
98 CPLString ETag{};
99 };
100
101 struct CachedDirList
102 {
103 bool bGotFileList = false;
104 unsigned int nGenerationAuthParameters = 0;
105 CPLStringList oFileList{}; /* only file name without path */
106 };
107
108 struct WriteFuncStruct
109 {
110 char *pBuffer = nullptr;
111 size_t nSize = 0;
112 bool bIsHTTP = false;
113 bool bMultiRange = false;
114 vsi_l_offset nStartOffset = 0;
115 vsi_l_offset nEndOffset = 0;
116 int nHTTPCode = 0;
117 vsi_l_offset nContentLength = 0;
118 bool bFoundContentRange = false;
119 bool bError = false;
120 bool bInterruptDownload = false;
121 bool bDetectRangeDownloadingError = false;
122 GIntBig nTimestampDate = 0; // Corresponds to Date: header field
123
124 VSILFILE *fp = nullptr;
125 VSICurlReadCbkFunc pfnReadCbk = nullptr;
126 void *pReadCbkUserData = nullptr;
127 bool bInterrupted = false;
128
129 #if !CURL_AT_LEAST_VERSION(7, 54, 0)
130 // Workaround to ignore extra HTTP response headers from
131 // proxies in older versions of curl.
132 // CURLOPT_SUPPRESS_CONNECT_HEADERS fixes this
133 bool bIsProxyConnectHeader = false;
134 #endif
135 };
136
137 struct PutData
138 {
139 const GByte *pabyData = nullptr;
140 size_t nOff = 0;
141 size_t nTotalSize = 0;
142
143 static size_t ReadCallBackBuffer(char *buffer, size_t size, size_t nitems,
144 void *instream)
145 {
146 PutData *poThis = static_cast<PutData *>(instream);
147 const size_t nSizeMax = size * nitems;
148 const size_t nSizeToWrite =
149 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
150 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
151 poThis->nOff += nSizeToWrite;
152 return nSizeToWrite;
153 }
154 };
155
156 /************************************************************************/
157 /* VSICurlFilesystemHandler */
158 /************************************************************************/
159
160 class VSICurlHandle;
161
162 class VSICurlFilesystemHandlerBase : public VSIFilesystemHandler
163 {
164 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBase)
165
166 struct FilenameOffsetPair
167 {
168 std::string filename_;
169 vsi_l_offset offset_;
170
171 FilenameOffsetPair(const std::string &filename, vsi_l_offset offset)
172 : filename_(filename), offset_(offset)
173 {
174 }
175
176 bool operator==(const FilenameOffsetPair &other) const
177 {
178 return filename_ == other.filename_ && offset_ == other.offset_;
179 }
180 };
181 struct FilenameOffsetPairHasher
182 {
183 std::size_t operator()(const FilenameOffsetPair &k) const
184 {
185 return std::hash<std::string>()(k.filename_) ^
186 std::hash<vsi_l_offset>()(k.offset_);
187 }
188 };
189
190 using RegionCacheType = lru11::Cache<
191 FilenameOffsetPair, std::shared_ptr<std::string>, lru11::NullLock,
192 std::unordered_map<
193 FilenameOffsetPair,
194 typename std::list<lru11::KeyValuePair<
195 FilenameOffsetPair, std::shared_ptr<std::string>>>::iterator,
196 FilenameOffsetPairHasher>>;
197
198 std::unique_ptr<RegionCacheType>
199 m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use
200 // GetRegionCache();
201 RegionCacheType *GetRegionCache();
202
203 // LRU cache that just keeps in memory if this file system handler is
204 // spposed to know the file properties of a file. The actual cache is a
205 // shared one among all network file systems.
206 // The aim of that design is that invalidating /vsis3/foo results in
207 // /vsis3_streaming/foo to be invalidated as well.
208 lru11::Cache<std::string, bool> oCacheFileProp;
209
210 int nCachedFilesInDirList = 0;
211 lru11::Cache<std::string, CachedDirList> oCacheDirList;
212
213 char **ParseHTMLFileList(const char *pszFilename, int nMaxFiles,
214 char *pszData, bool *pbGotFileList);
215
216 // Data structure and map to store regions that are in progress, to
217 // avoid simultaneous downloads of the same region in different threads
218 // Cf https://github.com/OSGeo/gdal/issues/8041
219 struct RegionInDownload
220 {
221 std::mutex oMutex{};
222 std::condition_variable oCond{};
223 bool bDownloadInProgress = false;
224 int nWaiters = 0;
225 std::string osData{};
226 };
227 std::mutex m_oMutex{};
228 std::map<std::string, std::unique_ptr<RegionInDownload>>
229 m_oMapRegionInDownload{};
230
231 protected:
232 CPLMutex *hMutex = nullptr;
233
234 virtual VSICurlHandle *CreateFileHandle(const char *pszFilename);
235 virtual char **GetFileList(const char *pszFilename, int nMaxFiles,
236 bool *pbGotFileList);
237
238 void RegisterEmptyDir(const CPLString &osDirname);
239
240 bool
241 AnalyseS3FileList(const CPLString &osBaseURL, const char *pszXML,
242 CPLStringList &osFileList, int nMaxFiles,
243 const std::set<std::string> &oSetIgnoredStorageClasses,
244 bool &bIsTruncated);
245
246 void AnalyseSwiftFileList(const CPLString &osBaseURL,
247 const CPLString &osPrefix, const char *pszJson,
248 CPLStringList &osFileList, int nMaxFilesThisQuery,
249 int nMaxFiles, bool &bIsTruncated,
250 CPLString &osNextMarker);
251
252 static const char *GetOptionsStatic();
253
254 VSICurlFilesystemHandlerBase();
255
256 public:
257 ~VSICurlFilesystemHandlerBase() override;
258
259 static bool IsAllowedFilename(const char *pszFilename);
260
261 VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess,
262 bool bSetError,
263 CSLConstList /* papszOptions */) override;
264
265 int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
266 int nFlags) override;
267 int Unlink(const char *pszFilename) override;
268 int Rename(const char *oldpath, const char *newpath) override;
269 int Mkdir(const char *pszDirname, long nMode) override;
270 int Rmdir(const char *pszDirname) override;
271 char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
272 char **SiblingFiles(const char *pszFilename) override;
273
274 int HasOptimizedReadMultiRange(const char * /* pszPath */) override
275 {
276 return true;
277 }
278
279 const char *GetActualURL(const char *pszFilename) override;
280
281 const char *GetOptions() override;
282
283 char **GetFileMetadata(const char *pszFilename, const char *pszDomain,
284 CSLConstList papszOptions) override;
285
286 char **ReadDirInternal(const char *pszDirname, int nMaxFiles,
287 bool *pbGotFileList);
288 void InvalidateDirContent(const char *pszDirname);
289
290 virtual const char *GetDebugKey() const = 0;
291
292 virtual CPLString GetFSPrefix() const = 0;
293 virtual bool AllowCachedDataFor(const char *pszFilename);
294
295 virtual bool IsLocal(const char * /* pszPath */) override
296 {
297 return false;
298 }
299 virtual bool
300 SupportsSequentialWrite(const char * /* pszPath */,
301 bool /* bAllowLocalTempFile */) override
302 {
303 return false;
304 }
305 virtual bool SupportsRandomWrite(const char * /* pszPath */,
306 bool /* bAllowLocalTempFile */) override
307 {
308 return false;
309 }
310
311 std::shared_ptr<std::string> GetRegion(const char *pszURL,
312 vsi_l_offset nFileOffsetStart);
313
314 void AddRegion(const char *pszURL, vsi_l_offset nFileOffsetStart,
315 size_t nSize, const char *pData);
316
317 std::string NotifyStartDownloadRegion(const std::string &osURL,
318 vsi_l_offset startOffset,
319 int nBlocks);
320 void NotifyStopDownloadRegion(const std::string &osURL,
321 vsi_l_offset startOffset, int nBlocks,
322 const std::string &osData);
323
324 bool GetCachedFileProp(const char *pszURL, FileProp &oFileProp);
325 void SetCachedFileProp(const char *pszURL, FileProp &oFileProp);
326 void InvalidateCachedData(const char *pszURL);
327
328 CURLM *GetCurlMultiHandleFor(const CPLString &osURL);
329
330 virtual void ClearCache();
331 virtual void PartialClearCache(const char *pszFilename);
332
333 bool GetCachedDirList(const char *pszURL, CachedDirList &oCachedDirList);
334 void SetCachedDirList(const char *pszURL, CachedDirList &oCachedDirList);
335 bool ExistsInCacheDirList(const CPLString &osDirname, bool *pbIsDir);
336
337 virtual CPLString GetURLFromFilename(const CPLString &osFilename);
338
339 std::string
340 GetStreamingFilename(const std::string &osFilename) const override = 0;
341
342 static std::set<std::string> GetS3IgnoredStorageClasses();
343 };
344
345 class VSICurlFilesystemHandler : public VSICurlFilesystemHandlerBase
346 {
347 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
348
349 public:
350 VSICurlFilesystemHandler() = default;
351
352 const char *GetDebugKey() const override
353 {
354 return "VSICURL";
355 }
356
357 CPLString GetFSPrefix() const override
358 {
359 return "/vsicurl/";
360 }
361
362 std::string
363 GetStreamingFilename(const std::string &osFilename) const override;
364 };
365
366 /************************************************************************/
367 /* VSICurlHandle */
368 /************************************************************************/
369
370 class VSICurlHandle : public VSIVirtualHandle
371 {
372 CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
373
374 protected:
375 VSICurlFilesystemHandlerBase *poFS = nullptr;
376
377 bool m_bCached = true;
378
379 mutable FileProp oFileProp{};
380
381 mutable std::mutex m_oMutex{};
382 CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
383 char *m_pszURL = nullptr; // e.g "http://example.com/foo"
384 mutable std::string m_osQueryString{}; // e.g. an Azure SAS
385
386 char **m_papszHTTPOptions = nullptr;
387
388 vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
389 int nBlocksToDownload = 1;
390
391 bool bStopOnInterruptUntilUninstall = false;
392 bool bInterrupted = false;
393 VSICurlReadCbkFunc pfnReadCbk = nullptr;
394 void *pReadCbkUserData = nullptr;
395
396 int m_nMaxRetry = 0;
397 double m_dfRetryDelay = 0.0;
398
399 CPLStringList m_aosHeaders{};
400
401 void DownloadRegionPostProcess(const vsi_l_offset startOffset,
402 const int nBlocks, const char *pBuffer,
403 size_t nSize);
404
405 private:
406 vsi_l_offset curOffset = 0;
407
408 bool bEOF = false;
409
410 virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
411
412 bool m_bUseHead = false;
413 bool m_bUseRedirectURLIfNoQueryStringParams = false;
414
415 // Specific to Planetary Computer signing:
416 // https://planetarycomputer.microsoft.com/docs/concepts/sas/
417 mutable bool m_bPlanetaryComputerURLSigning = false;
418 mutable std::string m_osPlanetaryComputerCollection{};
419 void ManagePlanetaryComputerSigning() const;
420
421 int ReadMultiRangeSingleGet(int nRanges, void **ppData,
422 const vsi_l_offset *panOffsets,
423 const size_t *panSizes);
424 CPLString GetRedirectURLIfValid(bool &bHasExpired) const;
425
426 void UpdateRedirectInfo(CURL *hCurlHandle,
427 const WriteFuncStruct &sWriteFuncHeaderData);
428
429 // Used by AdviseRead()
430 struct AdviseReadRange
431 {
432 bool bDone = false;
433 std::mutex oMutex{};
434 std::condition_variable oCV{};
435 vsi_l_offset nStartOffset = 0;
436 size_t nSize = 0;
437 std::vector<GByte> abyData{};
438 };
439 std::vector<std::unique_ptr<AdviseReadRange>> m_aoAdviseReadRanges{};
440 std::thread m_oThreadAdviseRead{};
441
442 protected:
443 virtual struct curl_slist *
444 GetCurlHeaders(const CPLString & /*osVerb*/,
445 const struct curl_slist * /* psExistingHeaders */)
446 {
447 return nullptr;
448 }
449 virtual bool AllowAutomaticRedirection()
450 {
451 return true;
452 }
453 virtual bool CanRestartOnError(const char *, const char *, bool)
454 {
455 return false;
456 }
457 virtual bool UseLimitRangeGetInsteadOfHead()
458 {
459 return false;
460 }
461 virtual bool IsDirectoryFromExists(const char * /*pszVerb*/,
462 int /*response_code*/)
463 {
464 return false;
465 }
466 virtual void ProcessGetFileSizeResult(const char * /* pszContent */)
467 {
468 }
469 void SetURL(const char *pszURL);
470 virtual bool Authenticate(const char * /* pszFilename */)
471 {
472 return false;
473 }
474
475 public:
476 VSICurlHandle(VSICurlFilesystemHandlerBase *poFS, const char *pszFilename,
477 const char *pszURLIn = nullptr);
478 ~VSICurlHandle() override;
479
480 int Seek(vsi_l_offset nOffset, int nWhence) override;
481 vsi_l_offset Tell() override;
482 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
483 int ReadMultiRange(int nRanges, void **ppData,
484 const vsi_l_offset *panOffsets,
485 const size_t *panSizes) override;
486 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
487 int Eof() override;
488 int Flush() override;
489 int Close() override;
490
491 bool HasPRead() const override
492 {
493 return true;
494 }
495 size_t PRead(void *pBuffer, size_t nSize,
496 vsi_l_offset nOffset) const override;
497
498 void AdviseRead(int nRanges, const vsi_l_offset *panOffsets,
499 const size_t *panSizes) override;
500
501 bool IsKnownFileSize() const
502 {
503 return oFileProp.bHasComputedFileSize;
504 }
505 vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
506 virtual vsi_l_offset GetFileSize(bool bSetError)
507 {
508 return GetFileSizeOrHeaders(bSetError, false);
509 }
510 bool Exists(bool bSetError);
511 bool IsDirectory() const
512 {
513 return oFileProp.bIsDirectory;
514 }
515 int GetMode() const
516 {
517 return oFileProp.nMode;
518 }
519 time_t GetMTime() const
520 {
521 return oFileProp.mTime;
522 }
523 const CPLStringList &GetHeaders()
524 {
525 return m_aosHeaders;
526 }
527
528 int InstallReadCbk(VSICurlReadCbkFunc pfnReadCbk, void *pfnUserData,
529 int bStopOnInterruptUntilUninstall);
530 int UninstallReadCbk();
531
532 const char *GetURL() const
533 {
534 return m_pszURL;
535 }
536 };
537
538 /************************************************************************/
539 /* VSICurlFilesystemHandlerBaseWritable */
540 /************************************************************************/
541
542 class VSICurlFilesystemHandlerBaseWritable : public VSICurlFilesystemHandlerBase
543 {
544 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBaseWritable)
545
546 protected:
547 VSICurlFilesystemHandlerBaseWritable() = default;
548
549 virtual VSIVirtualHandleUniquePtr
550 CreateWriteHandle(const char *pszFilename, CSLConstList papszOptions) = 0;
551
552 public:
553 VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess,
554 bool bSetError, CSLConstList papszOptions) override;
555
556 bool SupportsSequentialWrite(const char * /* pszPath */,
557 bool /* bAllowLocalTempFile */) override
558 {
559 return true;
560 }
561 bool SupportsRandomWrite(const char * /* pszPath */,
562 bool /* bAllowLocalTempFile */) override;
563 };
564
565 /************************************************************************/
566 /* IVSIS3LikeFSHandler */
567 /************************************************************************/
568
569 class IVSIS3LikeFSHandler : public VSICurlFilesystemHandlerBaseWritable
570 {
571 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
572
573 virtual int MkdirInternal(const char *pszDirname, long nMode,
574 bool bDoStatCheck);
575
576 protected:
577 char **GetFileList(const char *pszFilename, int nMaxFiles,
578 bool *pbGotFileList) override;
579
580 virtual IVSIS3LikeHandleHelper *CreateHandleHelper(const char *pszURI,
581 bool bAllowNoObject) = 0;
582
583 virtual int CopyObject(const char *oldpath, const char *newpath,
584 CSLConstList papszMetadata);
585
586 int RmdirRecursiveInternal(const char *pszDirname, int nBatchSize);
587
588 virtual bool
589 IsAllowedHeaderForObjectCreation(const char * /* pszHeaderName */)
590 {
591 return false;
592 }
593
594 IVSIS3LikeFSHandler() = default;
595
596 public:
597 int Unlink(const char *pszFilename) override;
598 int Mkdir(const char *pszDirname, long nMode) override;
599 int Rmdir(const char *pszDirname) override;
600 int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
601 int nFlags) override;
602 int Rename(const char *oldpath, const char *newpath) override;
603
604 virtual int CopyFile(const char *pszSource, const char *pszTarget,
605 VSILFILE *fpSource, vsi_l_offset nSourceSize,
606 const char *const *papszOptions,
607 GDALProgressFunc pProgressFunc,
608 void *pProgressData) override;
609
610 virtual int DeleteObject(const char *pszFilename);
611
612 bool Sync(const char *pszSource, const char *pszTarget,
613 const char *const *papszOptions, GDALProgressFunc pProgressFunc,
614 void *pProgressData, char ***ppapszOutputs) override;
615
616 VSIDIR *OpenDir(const char *pszPath, int nRecurseDepth,
617 const char *const *papszOptions) override;
618
619 // Multipart upload
620 virtual bool SupportsParallelMultipartUpload() const
621 {
622 return false;
623 }
624
625 virtual CPLString InitiateMultipartUpload(
626 const std::string &osFilename, IVSIS3LikeHandleHelper *poS3HandleHelper,
627 int nMaxRetry, double dfRetryDelay, CSLConstList papszOptions);
628 virtual CPLString UploadPart(const CPLString &osFilename, int nPartNumber,
629 const std::string &osUploadID,
630 vsi_l_offset nPosition, const void *pabyBuffer,
631 size_t nBufferSize,
632 IVSIS3LikeHandleHelper *poS3HandleHelper,
633 int nMaxRetry, double dfRetryDelay,
634 CSLConstList papszOptions);
635 virtual bool CompleteMultipart(const CPLString &osFilename,
636 const CPLString &osUploadID,
637 const std::vector<CPLString> &aosEtags,
638 vsi_l_offset nTotalSize,
639 IVSIS3LikeHandleHelper *poS3HandleHelper,
640 int nMaxRetry, double dfRetryDelay);
641 virtual bool AbortMultipart(const CPLString &osFilename,
642 const CPLString &osUploadID,
643 IVSIS3LikeHandleHelper *poS3HandleHelper,
644 int nMaxRetry, double dfRetryDelay);
645
646 bool AbortPendingUploads(const char *pszFilename) override;
647 };
648
649 /************************************************************************/
650 /* IVSIS3LikeHandle */
651 /************************************************************************/
652
653 class IVSIS3LikeHandle : public VSICurlHandle
654 {
655 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
656
657 protected:
658 bool UseLimitRangeGetInsteadOfHead() override
659 {
660 return true;
661 }
662 bool IsDirectoryFromExists(const char *pszVerb, int response_code) override
663 {
664 // A bit dirty, but on S3, a GET on a existing directory returns a 416
665 return response_code == 416 && EQUAL(pszVerb, "GET") &&
666 CPLString(m_pszURL).back() == '/';
667 }
668 void ProcessGetFileSizeResult(const char *pszContent) override
669 {
670 oFileProp.bIsDirectory =
671 strstr(pszContent, "ListBucketResult") != nullptr;
672 }
673
674 public:
675 IVSIS3LikeHandle(VSICurlFilesystemHandlerBase *poFSIn,
676 const char *pszFilename, const char *pszURLIn)
677 : VSICurlHandle(poFSIn, pszFilename, pszURLIn)
678 {
679 }
680 ~IVSIS3LikeHandle() override
681 {
682 }
683 };
684
685 /************************************************************************/
686 /* VSIS3WriteHandle */
687 /************************************************************************/
688
689 class VSIS3WriteHandle final : public VSIVirtualHandle
690 {
691 CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
692
693 IVSIS3LikeFSHandler *m_poFS = nullptr;
694 CPLString m_osFilename{};
695 IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
696 bool m_bUseChunked = false;
697 CPLStringList m_aosOptions{};
698 CPLStringList m_aosHTTPOptions{};
699
700 vsi_l_offset m_nCurOffset = 0;
701 int m_nBufferOff = 0;
702 int m_nBufferSize = 0;
703 bool m_bClosed = false;
704 GByte *m_pabyBuffer = nullptr;
705 CPLString m_osUploadID{};
706 int m_nPartNumber = 0;
707 std::vector<CPLString> m_aosEtags{};
708 bool m_bError = false;
709
710 CURLM *m_hCurlMulti = nullptr;
711 CURL *m_hCurl = nullptr;
712 const void *m_pBuffer = nullptr;
713 CPLString m_osCurlErrBuf{};
714 size_t m_nChunkedBufferOff = 0;
715 size_t m_nChunkedBufferSize = 0;
716 size_t m_nWrittenInPUT = 0;
717
718 int m_nMaxRetry = 0;
719 double m_dfRetryDelay = 0.0;
720 WriteFuncStruct m_sWriteFuncHeaderData{};
721
722 bool UploadPart();
723 bool DoSinglePartPUT();
724
725 static size_t ReadCallBackBufferChunked(char *buffer, size_t size,
726 size_t nitems, void *instream);
727 size_t WriteChunked(const void *pBuffer, size_t nSize, size_t nMemb);
728 int FinishChunkedTransfer();
729
730 void InvalidateParentDirectory();
731
732 public:
733 VSIS3WriteHandle(IVSIS3LikeFSHandler *poFS, const char *pszFilename,
734 IVSIS3LikeHandleHelper *poS3HandleHelper, bool bUseChunked,
735 CSLConstList papszOptions);
736 ~VSIS3WriteHandle() override;
737
738 int Seek(vsi_l_offset nOffset, int nWhence) override;
739 vsi_l_offset Tell() override;
740 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
741 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
742 int Eof() override;
743 int Close() override;
744
745 bool IsOK()
746 {
747 return m_bUseChunked || m_pabyBuffer != nullptr;
748 }
749 };
750
751 /************************************************************************/
752 /* VSIAppendWriteHandle */
753 /************************************************************************/
754
755 class VSIAppendWriteHandle : public VSIVirtualHandle
756 {
757 CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
758
759 protected:
760 VSICurlFilesystemHandlerBase *m_poFS = nullptr;
761 CPLString m_osFSPrefix{};
762 CPLString m_osFilename{};
763
764 vsi_l_offset m_nCurOffset = 0;
765 int m_nBufferOff = 0;
766 int m_nBufferSize = 0;
767 int m_nBufferOffReadCallback = 0;
768 bool m_bClosed = false;
769 GByte *m_pabyBuffer = nullptr;
770 bool m_bError = false;
771
772 static size_t ReadCallBackBuffer(char *buffer, size_t size, size_t nitems,
773 void *instream);
774 virtual bool Send(bool bIsLastBlock) = 0;
775
776 public:
777 VSIAppendWriteHandle(VSICurlFilesystemHandlerBase *poFS,
778 const char *pszFSPrefix, const char *pszFilename,
779 int nChunkSize);
780 virtual ~VSIAppendWriteHandle();
781
782 int Seek(vsi_l_offset nOffset, int nWhence) override;
783 vsi_l_offset Tell() override;
784 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
785 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
786 int Eof() override;
787 int Close() override;
788
789 bool IsOK()
790 {
791 return m_pabyBuffer != nullptr;
792 }
793 };
794
795 /************************************************************************/
796 /* VSIDIRWithMissingDirSynthesis */
797 /************************************************************************/
798
799 struct VSIDIRWithMissingDirSynthesis : public VSIDIR
800 {
801 std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
802
803 protected:
804 std::vector<std::string> m_aosSubpathsStack{};
805
806 void SynthetizeMissingDirectories(const std::string &osCurSubdir,
807 bool bAddEntryForThisSubdir);
808 };
809
810 /************************************************************************/
811 /* CurlRequestHelper */
812 /************************************************************************/
813
814 struct CurlRequestHelper
815 {
816 WriteFuncStruct sWriteFuncData{};
817 WriteFuncStruct sWriteFuncHeaderData{};
818 char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
819
820 CurlRequestHelper();
821 ~CurlRequestHelper();
822 long perform(CURL *hCurlHandle,
823 struct curl_slist *headers, // ownership transferred
824 VSICurlFilesystemHandlerBase *poFS,
825 IVSIS3LikeHandleHelper *poS3HandleHelper);
826 };
827
828 /************************************************************************/
829 /* NetworkStatisticsLogger */
830 /************************************************************************/
831
832 class NetworkStatisticsLogger
833 {
834 static int gnEnabled;
835 static NetworkStatisticsLogger gInstance;
836
837 NetworkStatisticsLogger() = default;
838
839 std::mutex m_mutex{};
840
841 struct Counters
842 {
843 GIntBig nHEAD = 0;
844 GIntBig nGET = 0;
845 GIntBig nPUT = 0;
846 GIntBig nPOST = 0;
847 GIntBig nDELETE = 0;
848 GIntBig nGETDownloadedBytes = 0;
849 GIntBig nPUTUploadedBytes = 0;
850 GIntBig nPOSTDownloadedBytes = 0;
851 GIntBig nPOSTUploadedBytes = 0;
852 };
853
854 enum class ContextPathType
855 {
856 FILESYSTEM,
857 FILE,
858 ACTION,
859 };
860
861 struct ContextPathItem
862 {
863 ContextPathType eType;
864 CPLString osName;
865
866 ContextPathItem(ContextPathType eTypeIn, const CPLString &osNameIn)
867 : eType(eTypeIn), osName(osNameIn)
868 {
869 }
870
871 bool operator<(const ContextPathItem &other) const
872 {
873 if (static_cast<int>(eType) < static_cast<int>(other.eType))
874 return true;
875 if (static_cast<int>(eType) > static_cast<int>(other.eType))
876 return false;
877 return osName < other.osName;
878 }
879 };
880
881 struct Stats
882 {
883 Counters counters{};
884 std::map<ContextPathItem, Stats> children{};
885
886 void AsJSON(CPLJSONObject &oJSON) const;
887 };
888
889 // Workaround bug in Coverity Scan
890 // coverity[generated_default_constructor_used_in_field_initializer]
891 Stats m_stats{};
892 std::map<GIntBig, std::vector<ContextPathItem>>
893 m_mapThreadIdToContextPath{};
894
895 static void ReadEnabled();
896
897 std::vector<Counters *> GetCountersForContext();
898
899 public:
900 static inline bool IsEnabled()
901 {
902 if (gnEnabled < 0)
903 {
904 ReadEnabled();
905 }
906 return gnEnabled == TRUE;
907 }
908
909 static void EnterFileSystem(const char *pszName);
910
911 static void LeaveFileSystem();
912
913 static void EnterFile(const char *pszName);
914
915 static void LeaveFile();
916
917 static void EnterAction(const char *pszName);
918
919 static void LeaveAction();
920
921 static void LogHEAD();
922
923 static void LogGET(size_t nDownloadedBytes);
924
925 static void LogPUT(size_t nUploadedBytes);
926
927 static void LogPOST(size_t nUploadedBytes, size_t nDownloadedBytes);
928
929 static void LogDELETE();
930
931 static void Reset();
932
933 static CPLString GetReportAsSerializedJSON();
934 };
935
936 struct NetworkStatisticsFileSystem
937 {
938 inline explicit NetworkStatisticsFileSystem(const char *pszName)
939 {
940 NetworkStatisticsLogger::EnterFileSystem(pszName);
941 }
942
943 inline ~NetworkStatisticsFileSystem()
944 {
945 NetworkStatisticsLogger::LeaveFileSystem();
946 }
947 };
948
949 struct NetworkStatisticsFile
950 {
951 inline explicit NetworkStatisticsFile(const char *pszName)
952 {
953 NetworkStatisticsLogger::EnterFile(pszName);
954 }
955
956 inline ~NetworkStatisticsFile()
957 {
958 NetworkStatisticsLogger::LeaveFile();
959 }
960 };
961
962 struct NetworkStatisticsAction
963 {
964 inline explicit NetworkStatisticsAction(const char *pszName)
965 {
966 NetworkStatisticsLogger::EnterAction(pszName);
967 }
968
969 inline ~NetworkStatisticsAction()
970 {
971 NetworkStatisticsLogger::LeaveAction();
972 }
973 };
974
975 int VSICURLGetDownloadChunkSize();
976
977 void VSICURLInitWriteFuncStruct(WriteFuncStruct *psStruct, VSILFILE *fp,
978 VSICurlReadCbkFunc pfnReadCbk,
979 void *pReadCbkUserData);
980 size_t VSICurlHandleWriteFunc(void *buffer, size_t count, size_t nmemb,
981 void *req);
982 void MultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle = nullptr);
983 void VSICURLResetHeaderAndWriterFunctions(CURL *hCurlHandle);
984
985 int VSICurlParseUnixPermissions(const char *pszPermissions);
986
987 // Cache of file properties (size, etc.)
988 bool VSICURLGetCachedFileProp(const char *pszURL, FileProp &oFileProp);
989 void VSICURLSetCachedFileProp(const char *pszURL, FileProp &oFileProp);
990 void VSICURLInvalidateCachedFileProp(const char *pszURL);
991 void VSICURLInvalidateCachedFilePropPrefix(const char *pszURL);
992 void VSICURLDestroyCacheFileProp();
993
994 } // namespace cpl
995
997
998 #endif // HAVE_CURL
999
1000 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
GByte
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:196
CPLStringList
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:437
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:311
EQUAL
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:562
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1178
CPLJSONObject
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:54
cpl_string.h
VSIVirtualHandle
Virtual file handle.
Definition: cpl_vsi_virtual.h:62
cpl_json.h
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:146
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:226
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:384
VSIStatBufL
#define VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:203
cpl_port.h
VSI_L_OFFSET_MAX
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:148
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1042