GDAL

cpl_vsil_curl_class.h
1/******************************************************************************
2 *
3 * Project: CPL - Common Portability Library
4 * Purpose: Declarations for /vsicurl/ and related file systems
5 * Author: Even Rouault, even.rouault at spatialys.com
6 *
7 ******************************************************************************
8 * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the "Software"),
12 * to deal in the Software without restriction, including without limitation
13 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 * and/or sell copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included
18 * in all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 ****************************************************************************/
28
29#ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30#define CPL_VSIL_CURL_CLASS_H_INCLUDED
31
32#ifdef HAVE_CURL
33
34#include "cpl_aws.h"
35#include "cpl_azure.h"
36#include "cpl_port.h"
37#include "cpl_json.h"
38#include "cpl_string.h"
39#include "cpl_vsil_curl_priv.h"
40#include "cpl_mem_cache.h"
41
42#include "cpl_curl_priv.h"
43
44#include <algorithm>
45#include <condition_variable>
46#include <set>
47#include <map>
48#include <memory>
49#include <mutex>
50#include <thread>
51#include <utility>
52
53// To avoid aliasing to CopyFile to CopyFileA on Windows
54#ifdef CopyFile
55#undef CopyFile
56#endif
57
59
60// Leave it for backward compatibility, but deprecate.
61#define HAVE_CURLINFO_REDIRECT_URL
62
63void VSICurlStreamingClearCache(void); // from cpl_vsil_curl_streaming.cpp
64
65struct curl_slist *VSICurlSetOptions(CURL *hCurlHandle, const char *pszURL,
66 const char *const *papszOptions);
67struct curl_slist *VSICurlMergeHeaders(struct curl_slist *poDest,
68 struct curl_slist *poSrcToDestroy);
69
70struct curl_slist *VSICurlSetContentTypeFromExt(struct curl_slist *polist,
71 const char *pszPath);
72
73struct curl_slist *VSICurlSetCreationHeadersFromOptions(
74 struct curl_slist *headers, CSLConstList papszOptions, const char *pszPath);
75
76namespace cpl
77{
78
79typedef enum
80{
81 EXIST_UNKNOWN = -1,
82 EXIST_NO,
83 EXIST_YES,
84} ExistStatus;
85
86class FileProp
87{
88 public:
89 unsigned int nGenerationAuthParameters = 0;
90 ExistStatus eExists = EXIST_UNKNOWN;
91 int nHTTPCode = 0;
92 vsi_l_offset fileSize = 0;
93 time_t mTime = 0;
94 time_t nExpireTimestampLocal = 0;
95 std::string osRedirectURL{};
96 bool bHasComputedFileSize = false;
97 bool bIsDirectory = false;
98 int nMode = 0; // st_mode member of struct stat
99 bool bS3LikeRedirect = false;
100 std::string ETag{};
101};
102
103struct CachedDirList
104{
105 bool bGotFileList = false;
106 unsigned int nGenerationAuthParameters = 0;
107 CPLStringList oFileList{}; /* only file name without path */
108};
109
110struct WriteFuncStruct
111{
112 char *pBuffer = nullptr;
113 size_t nSize = 0;
114 bool bIsHTTP = false;
115 bool bMultiRange = false;
116 vsi_l_offset nStartOffset = 0;
117 vsi_l_offset nEndOffset = 0;
118 int nHTTPCode = 0;
119 vsi_l_offset nContentLength = 0;
120 bool bFoundContentRange = false;
121 bool bError = false;
122 bool bInterruptDownload = false;
123 bool bDetectRangeDownloadingError = false;
124 GIntBig nTimestampDate = 0; // Corresponds to Date: header field
125
126 VSILFILE *fp = nullptr;
127 VSICurlReadCbkFunc pfnReadCbk = nullptr;
128 void *pReadCbkUserData = nullptr;
129 bool bInterrupted = false;
130};
131
132struct PutData
133{
134 const GByte *pabyData = nullptr;
135 size_t nOff = 0;
136 size_t nTotalSize = 0;
137
138 static size_t ReadCallBackBuffer(char *buffer, size_t size, size_t nitems,
139 void *instream)
140 {
141 PutData *poThis = static_cast<PutData *>(instream);
142 const size_t nSizeMax = size * nitems;
143 const size_t nSizeToWrite =
144 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
145 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
146 poThis->nOff += nSizeToWrite;
147 return nSizeToWrite;
148 }
149};
150
151/************************************************************************/
152/* VSICurlFilesystemHandler */
153/************************************************************************/
154
155class VSICurlHandle;
156
157class VSICurlFilesystemHandlerBase : public VSIFilesystemHandler
158{
159 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBase)
160
161 struct FilenameOffsetPair
162 {
163 std::string filename_;
164 vsi_l_offset offset_;
165
166 FilenameOffsetPair(const std::string &filename, vsi_l_offset offset)
167 : filename_(filename), offset_(offset)
168 {
169 }
170
171 bool operator==(const FilenameOffsetPair &other) const
172 {
173 return filename_ == other.filename_ && offset_ == other.offset_;
174 }
175 };
176
177 struct FilenameOffsetPairHasher
178 {
179 std::size_t operator()(const FilenameOffsetPair &k) const
180 {
181 return std::hash<std::string>()(k.filename_) ^
182 std::hash<vsi_l_offset>()(k.offset_);
183 }
184 };
185
186 using RegionCacheType = lru11::Cache<
187 FilenameOffsetPair, std::shared_ptr<std::string>, lru11::NullLock,
188 std::unordered_map<
189 FilenameOffsetPair,
190 typename std::list<lru11::KeyValuePair<
191 FilenameOffsetPair, std::shared_ptr<std::string>>>::iterator,
192 FilenameOffsetPairHasher>>;
193
194 std::unique_ptr<RegionCacheType>
195 m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use
196 // GetRegionCache();
197 RegionCacheType *GetRegionCache();
198
199 // LRU cache that just keeps in memory if this file system handler is
200 // spposed to know the file properties of a file. The actual cache is a
201 // shared one among all network file systems.
202 // The aim of that design is that invalidating /vsis3/foo results in
203 // /vsis3_streaming/foo to be invalidated as well.
204 lru11::Cache<std::string, bool> oCacheFileProp;
205
206 int nCachedFilesInDirList = 0;
207 lru11::Cache<std::string, CachedDirList> oCacheDirList;
208
209 char **ParseHTMLFileList(const char *pszFilename, int nMaxFiles,
210 char *pszData, bool *pbGotFileList);
211
212 // Data structure and map to store regions that are in progress, to
213 // avoid simultaneous downloads of the same region in different threads
214 // Cf https://github.com/OSGeo/gdal/issues/8041
215 struct RegionInDownload
216 {
217 std::mutex oMutex{};
218 std::condition_variable oCond{};
219 bool bDownloadInProgress = false;
220 int nWaiters = 0;
221 std::string osData{};
222 };
223
224 std::mutex m_oMutex{};
225 std::map<std::string, std::unique_ptr<RegionInDownload>>
226 m_oMapRegionInDownload{};
227
228 protected:
229 CPLMutex *hMutex = nullptr;
230
231 virtual VSICurlHandle *CreateFileHandle(const char *pszFilename);
232 virtual char **GetFileList(const char *pszFilename, int nMaxFiles,
233 bool *pbGotFileList);
234
235 void RegisterEmptyDir(const std::string &osDirname);
236
237 bool
238 AnalyseS3FileList(const std::string &osBaseURL, const char *pszXML,
239 CPLStringList &osFileList, int nMaxFiles,
240 const std::set<std::string> &oSetIgnoredStorageClasses,
241 bool &bIsTruncated);
242
243 void AnalyseSwiftFileList(const std::string &osBaseURL,
244 const std::string &osPrefix, const char *pszJson,
245 CPLStringList &osFileList, int nMaxFilesThisQuery,
246 int nMaxFiles, bool &bIsTruncated,
247 std::string &osNextMarker);
248
249 static const char *GetOptionsStatic();
250
251 VSICurlFilesystemHandlerBase();
252
253 public:
254 ~VSICurlFilesystemHandlerBase() override;
255
256 static bool IsAllowedFilename(const char *pszFilename);
257
258 VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess,
259 bool bSetError,
260 CSLConstList /* papszOptions */) override;
261
262 int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
263 int nFlags) override;
264 int Unlink(const char *pszFilename) override;
265 int Rename(const char *oldpath, const char *newpath) override;
266 int Mkdir(const char *pszDirname, long nMode) override;
267 int Rmdir(const char *pszDirname) override;
268 char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
269 char **SiblingFiles(const char *pszFilename) override;
270
271 int HasOptimizedReadMultiRange(const char * /* pszPath */) override
272 {
273 return true;
274 }
275
276 const char *GetActualURL(const char *pszFilename) override;
277
278 const char *GetOptions() override;
279
280 char **GetFileMetadata(const char *pszFilename, const char *pszDomain,
281 CSLConstList papszOptions) override;
282
283 char **ReadDirInternal(const char *pszDirname, int nMaxFiles,
284 bool *pbGotFileList);
285 void InvalidateDirContent(const char *pszDirname);
286
287 virtual const char *GetDebugKey() const = 0;
288
289 virtual std::string GetFSPrefix() const = 0;
290 virtual bool AllowCachedDataFor(const char *pszFilename);
291
292 virtual bool IsLocal(const char * /* pszPath */) override
293 {
294 return false;
295 }
296
297 virtual bool
298 SupportsSequentialWrite(const char * /* pszPath */,
299 bool /* bAllowLocalTempFile */) override
300 {
301 return false;
302 }
303
304 virtual bool SupportsRandomWrite(const char * /* pszPath */,
305 bool /* bAllowLocalTempFile */) override
306 {
307 return false;
308 }
309
310 std::shared_ptr<std::string> GetRegion(const char *pszURL,
311 vsi_l_offset nFileOffsetStart);
312
313 void AddRegion(const char *pszURL, vsi_l_offset nFileOffsetStart,
314 size_t nSize, const char *pData);
315
316 std::pair<bool, std::string>
317 NotifyStartDownloadRegion(const std::string &osURL,
318 vsi_l_offset startOffset, int nBlocks);
319 void NotifyStopDownloadRegion(const std::string &osURL,
320 vsi_l_offset startOffset, int nBlocks,
321 const std::string &osData);
322
323 bool GetCachedFileProp(const char *pszURL, FileProp &oFileProp);
324 void SetCachedFileProp(const char *pszURL, FileProp &oFileProp);
325 void InvalidateCachedData(const char *pszURL);
326
327 CURLM *GetCurlMultiHandleFor(const std::string &osURL);
328
329 virtual void ClearCache();
330 virtual void PartialClearCache(const char *pszFilename);
331
332 bool GetCachedDirList(const char *pszURL, CachedDirList &oCachedDirList);
333 void SetCachedDirList(const char *pszURL, CachedDirList &oCachedDirList);
334 bool ExistsInCacheDirList(const std::string &osDirname, bool *pbIsDir);
335
336 virtual std::string GetURLFromFilename(const std::string &osFilename);
337
338 std::string
339 GetStreamingFilename(const std::string &osFilename) const override = 0;
340
341 static std::set<std::string> GetS3IgnoredStorageClasses();
342};
343
344class VSICurlFilesystemHandler : public VSICurlFilesystemHandlerBase
345{
346 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
347
348 public:
349 VSICurlFilesystemHandler() = default;
350
351 const char *GetDebugKey() const override
352 {
353 return "VSICURL";
354 }
355
356 std::string GetFSPrefix() const override
357 {
358 return "/vsicurl/";
359 }
360
361 std::string
362 GetStreamingFilename(const std::string &osFilename) const override;
363};
364
365/************************************************************************/
366/* VSICurlHandle */
367/************************************************************************/
368
369class VSICurlHandle : public VSIVirtualHandle
370{
371 CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
372
373 protected:
374 VSICurlFilesystemHandlerBase *poFS = nullptr;
375
376 bool m_bCached = true;
377
378 mutable FileProp oFileProp{};
379
380 mutable std::mutex m_oMutex{};
381 std::string m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
382 char *m_pszURL = nullptr; // e.g "http://example.com/foo"
383 mutable std::string m_osQueryString{}; // e.g. an Azure SAS
384
385 char **m_papszHTTPOptions = nullptr;
386
387 vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
388 int nBlocksToDownload = 1;
389
390 bool bStopOnInterruptUntilUninstall = false;
391 bool bInterrupted = false;
392 VSICurlReadCbkFunc pfnReadCbk = nullptr;
393 void *pReadCbkUserData = nullptr;
394
395 int m_nMaxRetry = 0;
396 double m_dfRetryDelay = 0.0;
397
398 CPLStringList m_aosHeaders{};
399
400 void DownloadRegionPostProcess(const vsi_l_offset startOffset,
401 const int nBlocks, const char *pBuffer,
402 size_t nSize);
403
404 private:
405 vsi_l_offset curOffset = 0;
406
407 bool bEOF = false;
408
409 virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
410
411 bool m_bUseHead = false;
412 bool m_bUseRedirectURLIfNoQueryStringParams = false;
413
414 // Specific to Planetary Computer signing:
415 // https://planetarycomputer.microsoft.com/docs/concepts/sas/
416 mutable bool m_bPlanetaryComputerURLSigning = false;
417 mutable std::string m_osPlanetaryComputerCollection{};
418 void ManagePlanetaryComputerSigning() const;
419
420 int ReadMultiRangeSingleGet(int nRanges, void **ppData,
421 const vsi_l_offset *panOffsets,
422 const size_t *panSizes);
423 std::string GetRedirectURLIfValid(bool &bHasExpired) const;
424
425 void UpdateRedirectInfo(CURL *hCurlHandle,
426 const WriteFuncStruct &sWriteFuncHeaderData);
427
428 // Used by AdviseRead()
429 struct AdviseReadRange
430 {
431 bool bDone = false;
432 std::mutex oMutex{};
433 std::condition_variable oCV{};
434 vsi_l_offset nStartOffset = 0;
435 size_t nSize = 0;
436 std::vector<GByte> abyData{};
437 };
438
439 std::vector<std::unique_ptr<AdviseReadRange>> m_aoAdviseReadRanges{};
440 std::thread m_oThreadAdviseRead{};
441
442 protected:
443 virtual struct curl_slist *
444 GetCurlHeaders(const std::string & /*osVerb*/,
445 const struct curl_slist * /* psExistingHeaders */)
446 {
447 return nullptr;
448 }
449
450 virtual bool AllowAutomaticRedirection()
451 {
452 return true;
453 }
454
455 virtual bool CanRestartOnError(const char *, const char *, bool)
456 {
457 return false;
458 }
459
460 virtual bool UseLimitRangeGetInsteadOfHead()
461 {
462 return false;
463 }
464
465 virtual bool IsDirectoryFromExists(const char * /*pszVerb*/,
466 int /*response_code*/)
467 {
468 return false;
469 }
470
471 virtual void ProcessGetFileSizeResult(const char * /* pszContent */)
472 {
473 }
474
475 void SetURL(const char *pszURL);
476
477 virtual bool Authenticate(const char * /* pszFilename */)
478 {
479 return false;
480 }
481
482 public:
483 VSICurlHandle(VSICurlFilesystemHandlerBase *poFS, const char *pszFilename,
484 const char *pszURLIn = nullptr);
485 ~VSICurlHandle() override;
486
487 int Seek(vsi_l_offset nOffset, int nWhence) override;
488 vsi_l_offset Tell() override;
489 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
490 int ReadMultiRange(int nRanges, void **ppData,
491 const vsi_l_offset *panOffsets,
492 const size_t *panSizes) override;
493 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
494 int Eof() override;
495 int Flush() override;
496 int Close() override;
497
498 bool HasPRead() const override
499 {
500 return true;
501 }
502
503 size_t PRead(void *pBuffer, size_t nSize,
504 vsi_l_offset nOffset) const override;
505
506 void AdviseRead(int nRanges, const vsi_l_offset *panOffsets,
507 const size_t *panSizes) override;
508
509 size_t GetAdviseReadTotalBytesLimit() const override;
510
511 bool IsKnownFileSize() const
512 {
513 return oFileProp.bHasComputedFileSize;
514 }
515
516 vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
517
518 virtual vsi_l_offset GetFileSize(bool bSetError)
519 {
520 return GetFileSizeOrHeaders(bSetError, false);
521 }
522
523 bool Exists(bool bSetError);
524
525 bool IsDirectory() const
526 {
527 return oFileProp.bIsDirectory;
528 }
529
530 int GetMode() const
531 {
532 return oFileProp.nMode;
533 }
534
535 time_t GetMTime() const
536 {
537 return oFileProp.mTime;
538 }
539
540 const CPLStringList &GetHeaders()
541 {
542 return m_aosHeaders;
543 }
544
545 int InstallReadCbk(VSICurlReadCbkFunc pfnReadCbk, void *pfnUserData,
546 int bStopOnInterruptUntilUninstall);
547 int UninstallReadCbk();
548
549 const char *GetURL() const
550 {
551 return m_pszURL;
552 }
553};
554
555/************************************************************************/
556/* VSICurlFilesystemHandlerBaseWritable */
557/************************************************************************/
558
559class VSICurlFilesystemHandlerBaseWritable : public VSICurlFilesystemHandlerBase
560{
561 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBaseWritable)
562
563 protected:
564 VSICurlFilesystemHandlerBaseWritable() = default;
565
566 virtual VSIVirtualHandleUniquePtr
567 CreateWriteHandle(const char *pszFilename, CSLConstList papszOptions) = 0;
568
569 public:
570 VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess,
571 bool bSetError, CSLConstList papszOptions) override;
572
573 bool SupportsSequentialWrite(const char * /* pszPath */,
574 bool /* bAllowLocalTempFile */) override
575 {
576 return true;
577 }
578
579 bool SupportsRandomWrite(const char * /* pszPath */,
580 bool /* bAllowLocalTempFile */) override;
581};
582
583/************************************************************************/
584/* IVSIS3LikeFSHandler */
585/************************************************************************/
586
587class IVSIS3LikeFSHandler : public VSICurlFilesystemHandlerBaseWritable
588{
589 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
590
591 virtual int MkdirInternal(const char *pszDirname, long nMode,
592 bool bDoStatCheck);
593
594 protected:
595 char **GetFileList(const char *pszFilename, int nMaxFiles,
596 bool *pbGotFileList) override;
597
598 virtual IVSIS3LikeHandleHelper *CreateHandleHelper(const char *pszURI,
599 bool bAllowNoObject) = 0;
600
601 virtual int CopyObject(const char *oldpath, const char *newpath,
602 CSLConstList papszMetadata);
603
604 int RmdirRecursiveInternal(const char *pszDirname, int nBatchSize);
605
606 virtual bool
607 IsAllowedHeaderForObjectCreation(const char * /* pszHeaderName */)
608 {
609 return false;
610 }
611
612 IVSIS3LikeFSHandler() = default;
613
614 public:
615 int Unlink(const char *pszFilename) override;
616 int Mkdir(const char *pszDirname, long nMode) override;
617 int Rmdir(const char *pszDirname) override;
618 int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
619 int nFlags) override;
620 int Rename(const char *oldpath, const char *newpath) override;
621
622 virtual int CopyFile(const char *pszSource, const char *pszTarget,
623 VSILFILE *fpSource, vsi_l_offset nSourceSize,
624 const char *const *papszOptions,
625 GDALProgressFunc pProgressFunc,
626 void *pProgressData) override;
627
628 virtual int DeleteObject(const char *pszFilename);
629
630 virtual int *DeleteObjectBatch(CSLConstList papszFilesOrDirs);
631
632 bool Sync(const char *pszSource, const char *pszTarget,
633 const char *const *papszOptions, GDALProgressFunc pProgressFunc,
634 void *pProgressData, char ***ppapszOutputs) override;
635
636 VSIDIR *OpenDir(const char *pszPath, int nRecurseDepth,
637 const char *const *papszOptions) override;
638
639 // Multipart upload
640 virtual bool SupportsParallelMultipartUpload() const
641 {
642 return false;
643 }
644
645 virtual std::string InitiateMultipartUpload(
646 const std::string &osFilename, IVSIS3LikeHandleHelper *poS3HandleHelper,
647 int nMaxRetry, double dfRetryDelay, CSLConstList papszOptions);
648 virtual std::string
649 UploadPart(const std::string &osFilename, int nPartNumber,
650 const std::string &osUploadID, vsi_l_offset nPosition,
651 const void *pabyBuffer, size_t nBufferSize,
652 IVSIS3LikeHandleHelper *poS3HandleHelper, int nMaxRetry,
653 double dfRetryDelay, CSLConstList papszOptions);
654 virtual bool CompleteMultipart(const std::string &osFilename,
655 const std::string &osUploadID,
656 const std::vector<std::string> &aosEtags,
657 vsi_l_offset nTotalSize,
658 IVSIS3LikeHandleHelper *poS3HandleHelper,
659 int nMaxRetry, double dfRetryDelay);
660 virtual bool AbortMultipart(const std::string &osFilename,
661 const std::string &osUploadID,
662 IVSIS3LikeHandleHelper *poS3HandleHelper,
663 int nMaxRetry, double dfRetryDelay);
664
665 bool AbortPendingUploads(const char *pszFilename) override;
666};
667
668/************************************************************************/
669/* IVSIS3LikeHandle */
670/************************************************************************/
671
672class IVSIS3LikeHandle : public VSICurlHandle
673{
674 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
675
676 protected:
677 bool UseLimitRangeGetInsteadOfHead() override
678 {
679 return true;
680 }
681
682 bool IsDirectoryFromExists(const char *pszVerb, int response_code) override
683 {
684 // A bit dirty, but on S3, a GET on a existing directory returns a 416
685 return response_code == 416 && EQUAL(pszVerb, "GET") &&
686 std::string(m_pszURL).back() == '/';
687 }
688
689 void ProcessGetFileSizeResult(const char *pszContent) override
690 {
691 oFileProp.bIsDirectory =
692 strstr(pszContent, "ListBucketResult") != nullptr;
693 }
694
695 public:
696 IVSIS3LikeHandle(VSICurlFilesystemHandlerBase *poFSIn,
697 const char *pszFilename, const char *pszURLIn)
698 : VSICurlHandle(poFSIn, pszFilename, pszURLIn)
699 {
700 }
701
702 ~IVSIS3LikeHandle() override
703 {
704 }
705};
706
707/************************************************************************/
708/* VSIS3WriteHandle */
709/************************************************************************/
710
711class VSIS3WriteHandle final : public VSIVirtualHandle
712{
713 CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
714
715 IVSIS3LikeFSHandler *m_poFS = nullptr;
716 std::string m_osFilename{};
717 IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
718 bool m_bUseChunked = false;
719 CPLStringList m_aosOptions{};
720 CPLStringList m_aosHTTPOptions{};
721
722 vsi_l_offset m_nCurOffset = 0;
723 int m_nBufferOff = 0;
724 int m_nBufferSize = 0;
725 bool m_bClosed = false;
726 GByte *m_pabyBuffer = nullptr;
727 std::string m_osUploadID{};
728 int m_nPartNumber = 0;
729 std::vector<std::string> m_aosEtags{};
730 bool m_bError = false;
731
732 CURLM *m_hCurlMulti = nullptr;
733 CURL *m_hCurl = nullptr;
734 const void *m_pBuffer = nullptr;
735 std::string m_osCurlErrBuf{};
736 size_t m_nChunkedBufferOff = 0;
737 size_t m_nChunkedBufferSize = 0;
738 size_t m_nWrittenInPUT = 0;
739
740 int m_nMaxRetry = 0;
741 double m_dfRetryDelay = 0.0;
742 WriteFuncStruct m_sWriteFuncHeaderData{};
743
744 bool UploadPart();
745 bool DoSinglePartPUT();
746
747 static size_t ReadCallBackBufferChunked(char *buffer, size_t size,
748 size_t nitems, void *instream);
749 size_t WriteChunked(const void *pBuffer, size_t nSize, size_t nMemb);
750 int FinishChunkedTransfer();
751
752 void InvalidateParentDirectory();
753
754 public:
755 VSIS3WriteHandle(IVSIS3LikeFSHandler *poFS, const char *pszFilename,
756 IVSIS3LikeHandleHelper *poS3HandleHelper, bool bUseChunked,
757 CSLConstList papszOptions);
758 ~VSIS3WriteHandle() override;
759
760 int Seek(vsi_l_offset nOffset, int nWhence) override;
761 vsi_l_offset Tell() override;
762 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
763 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
764 int Eof() override;
765 int Close() override;
766
767 bool IsOK()
768 {
769 return m_bUseChunked || m_pabyBuffer != nullptr;
770 }
771};
772
773/************************************************************************/
774/* VSIAppendWriteHandle */
775/************************************************************************/
776
777class VSIAppendWriteHandle : public VSIVirtualHandle
778{
779 CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
780
781 protected:
782 VSICurlFilesystemHandlerBase *m_poFS = nullptr;
783 std::string m_osFSPrefix{};
784 std::string m_osFilename{};
785
786 vsi_l_offset m_nCurOffset = 0;
787 int m_nBufferOff = 0;
788 int m_nBufferSize = 0;
789 int m_nBufferOffReadCallback = 0;
790 bool m_bClosed = false;
791 GByte *m_pabyBuffer = nullptr;
792 bool m_bError = false;
793
794 static size_t ReadCallBackBuffer(char *buffer, size_t size, size_t nitems,
795 void *instream);
796 virtual bool Send(bool bIsLastBlock) = 0;
797
798 public:
799 VSIAppendWriteHandle(VSICurlFilesystemHandlerBase *poFS,
800 const char *pszFSPrefix, const char *pszFilename,
801 int nChunkSize);
802 virtual ~VSIAppendWriteHandle();
803
804 int Seek(vsi_l_offset nOffset, int nWhence) override;
805 vsi_l_offset Tell() override;
806 size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
807 size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
808 int Eof() override;
809 int Close() override;
810
811 bool IsOK()
812 {
813 return m_pabyBuffer != nullptr;
814 }
815};
816
817/************************************************************************/
818/* VSIDIRWithMissingDirSynthesis */
819/************************************************************************/
820
821struct VSIDIRWithMissingDirSynthesis : public VSIDIR
822{
823 std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
824
825 protected:
826 std::vector<std::string> m_aosSubpathsStack{};
827
828 void SynthetizeMissingDirectories(const std::string &osCurSubdir,
829 bool bAddEntryForThisSubdir);
830};
831
832/************************************************************************/
833/* CurlRequestHelper */
834/************************************************************************/
835
836struct CurlRequestHelper
837{
838 WriteFuncStruct sWriteFuncData{};
839 WriteFuncStruct sWriteFuncHeaderData{};
840 char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
841
842 CurlRequestHelper();
843 ~CurlRequestHelper();
844 long perform(CURL *hCurlHandle,
845 struct curl_slist *headers, // ownership transferred
846 VSICurlFilesystemHandlerBase *poFS,
847 IVSIS3LikeHandleHelper *poS3HandleHelper);
848};
849
850/************************************************************************/
851/* NetworkStatisticsLogger */
852/************************************************************************/
853
854class NetworkStatisticsLogger
855{
856 static int gnEnabled;
857 static NetworkStatisticsLogger gInstance;
858
859 NetworkStatisticsLogger() = default;
860
861 std::mutex m_mutex{};
862
863 struct Counters
864 {
865 GIntBig nHEAD = 0;
866 GIntBig nGET = 0;
867 GIntBig nPUT = 0;
868 GIntBig nPOST = 0;
869 GIntBig nDELETE = 0;
870 GIntBig nGETDownloadedBytes = 0;
871 GIntBig nPUTUploadedBytes = 0;
872 GIntBig nPOSTDownloadedBytes = 0;
873 GIntBig nPOSTUploadedBytes = 0;
874 };
875
876 enum class ContextPathType
877 {
878 FILESYSTEM,
879 FILE,
880 ACTION,
881 };
882
883 struct ContextPathItem
884 {
885 ContextPathType eType;
886 std::string osName;
887
888 ContextPathItem(ContextPathType eTypeIn, const std::string &osNameIn)
889 : eType(eTypeIn), osName(osNameIn)
890 {
891 }
892
893 bool operator<(const ContextPathItem &other) const
894 {
895 if (static_cast<int>(eType) < static_cast<int>(other.eType))
896 return true;
897 if (static_cast<int>(eType) > static_cast<int>(other.eType))
898 return false;
899 return osName < other.osName;
900 }
901 };
902
903 struct Stats
904 {
905 Counters counters{};
906 std::map<ContextPathItem, Stats> children{};
907
908 void AsJSON(CPLJSONObject &oJSON) const;
909 };
910
911 // Workaround bug in Coverity Scan
912 // coverity[generated_default_constructor_used_in_field_initializer]
913 Stats m_stats{};
914 std::map<GIntBig, std::vector<ContextPathItem>>
915 m_mapThreadIdToContextPath{};
916
917 static void ReadEnabled();
918
919 std::vector<Counters *> GetCountersForContext();
920
921 public:
922 static inline bool IsEnabled()
923 {
924 if (gnEnabled < 0)
925 {
926 ReadEnabled();
927 }
928 return gnEnabled == TRUE;
929 }
930
931 static void EnterFileSystem(const char *pszName);
932
933 static void LeaveFileSystem();
934
935 static void EnterFile(const char *pszName);
936
937 static void LeaveFile();
938
939 static void EnterAction(const char *pszName);
940
941 static void LeaveAction();
942
943 static void LogHEAD();
944
945 static void LogGET(size_t nDownloadedBytes);
946
947 static void LogPUT(size_t nUploadedBytes);
948
949 static void LogPOST(size_t nUploadedBytes, size_t nDownloadedBytes);
950
951 static void LogDELETE();
952
953 static void Reset();
954
955 static std::string GetReportAsSerializedJSON();
956};
957
958struct NetworkStatisticsFileSystem
959{
960 inline explicit NetworkStatisticsFileSystem(const char *pszName)
961 {
962 NetworkStatisticsLogger::EnterFileSystem(pszName);
963 }
964
965 inline ~NetworkStatisticsFileSystem()
966 {
967 NetworkStatisticsLogger::LeaveFileSystem();
968 }
969};
970
971struct NetworkStatisticsFile
972{
973 inline explicit NetworkStatisticsFile(const char *pszName)
974 {
975 NetworkStatisticsLogger::EnterFile(pszName);
976 }
977
978 inline ~NetworkStatisticsFile()
979 {
980 NetworkStatisticsLogger::LeaveFile();
981 }
982};
983
984struct NetworkStatisticsAction
985{
986 inline explicit NetworkStatisticsAction(const char *pszName)
987 {
988 NetworkStatisticsLogger::EnterAction(pszName);
989 }
990
991 inline ~NetworkStatisticsAction()
992 {
993 NetworkStatisticsLogger::LeaveAction();
994 }
995};
996
997} // namespace cpl
998
999int VSICURLGetDownloadChunkSize();
1000
1001void VSICURLInitWriteFuncStruct(cpl::WriteFuncStruct *psStruct, VSILFILE *fp,
1002 VSICurlReadCbkFunc pfnReadCbk,
1003 void *pReadCbkUserData);
1004size_t VSICurlHandleWriteFunc(void *buffer, size_t count, size_t nmemb,
1005 void *req);
1006void VSICURLMultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle = nullptr);
1007void VSICURLResetHeaderAndWriterFunctions(CURL *hCurlHandle);
1008
1009int VSICurlParseUnixPermissions(const char *pszPermissions);
1010
1011// Cache of file properties (size, etc.)
1012bool VSICURLGetCachedFileProp(const char *pszURL, cpl::FileProp &oFileProp);
1013void VSICURLSetCachedFileProp(const char *pszURL, cpl::FileProp &oFileProp);
1014void VSICURLInvalidateCachedFileProp(const char *pszURL);
1015void VSICURLInvalidateCachedFilePropPrefix(const char *pszURL);
1016void VSICURLDestroyCacheFileProp();
1017
1018void VSICURLMultiCleanup(CURLM *hCurlMultiHandle);
1019
1021
1022#endif // HAVE_CURL
1023
1024#endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
CPLJSONObject
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:57
CPLStringList
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:449
cpl_json.h
Interface for read and write JSON documents.
cpl_port.h
Core portability definitions for CPL.
EQUAL
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:551
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1042
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1183
GByte
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:185
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:215
cpl_string.h
Various convenience functions for working with strings and string lists.
VSIStatBufL
#define VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:203
VSI_L_OFFSET_MAX
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:148
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:402
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:146
VSIVirtualHandle
Virtual file handle.
Definition: cpl_vsi_virtual.h:63