GDAL

cpl_vsi_virtual.h
1/******************************************************************************
2 * $Id$
3 *
4 * Project: VSI Virtual File System
5 * Purpose: Declarations for classes related to the virtual filesystem.
6 * These would only be normally required by applications implementing
7 * their own virtual file system classes which should be rare.
8 * The class interface may be fragile through versions.
9 * Author: Frank Warmerdam, warmerdam@pobox.com
10 *
11 ******************************************************************************
12 * Copyright (c) 2005, Frank Warmerdam <warmerdam@pobox.com>
13 * Copyright (c) 2010-2014, Even Rouault <even dot rouault at spatialys.com>
14 *
15 * SPDX-License-Identifier: MIT
16 ****************************************************************************/
17
18#ifndef CPL_VSI_VIRTUAL_H_INCLUDED
19#define CPL_VSI_VIRTUAL_H_INCLUDED
20
21#include "cpl_vsi.h"
22#include "cpl_vsi_error.h"
23#include "cpl_string.h"
24#include "cpl_multiproc.h"
25
26#include <map>
27#include <memory>
28#include <vector>
29#include <string>
30
31// To avoid aliasing to GetDiskFreeSpace to GetDiskFreeSpaceA on Windows
32#ifdef GetDiskFreeSpace
33#undef GetDiskFreeSpace
34#endif
35
36// To avoid aliasing to CopyFile to CopyFileA on Windows
37#ifdef CopyFile
38#undef CopyFile
39#endif
40
41/************************************************************************/
42/* VSIVirtualHandle */
43/************************************************************************/
44
46 struct CPL_DLL VSIVirtualHandle
47{
48 public:
49 virtual int Seek(vsi_l_offset nOffset, int nWhence) = 0;
50 virtual vsi_l_offset Tell() = 0;
51 virtual size_t Read(void *pBuffer, size_t nSize, size_t nCount) = 0;
52 virtual int ReadMultiRange(int nRanges, void **ppData,
53 const vsi_l_offset *panOffsets,
54 const size_t *panSizes);
55
68 virtual void AdviseRead(CPL_UNUSED int nRanges,
69 CPL_UNUSED const vsi_l_offset *panOffsets,
70 CPL_UNUSED const size_t *panSizes)
71 {
72 }
73
86 virtual size_t GetAdviseReadTotalBytesLimit() const
87 {
88 return 0;
89 }
90
91 virtual size_t Write(const void *pBuffer, size_t nSize, size_t nCount) = 0;
92
93 int Printf(CPL_FORMAT_STRING(const char *pszFormat), ...)
94 CPL_PRINT_FUNC_FORMAT(2, 3);
95
96 virtual void ClearErr() = 0;
97
98 virtual int Eof() = 0;
99
100 virtual int Error() = 0;
101
102 virtual int Flush()
103 {
104 return 0;
105 }
106
107 virtual int Close() = 0;
108 // Base implementation that only supports file extension.
109 virtual int Truncate(vsi_l_offset nNewSize);
110
111 virtual void *GetNativeFileDescriptor()
112 {
113 return nullptr;
114 }
115
116 virtual VSIRangeStatus GetRangeStatus(CPL_UNUSED vsi_l_offset nOffset,
117 CPL_UNUSED vsi_l_offset nLength)
118 {
119 return VSI_RANGE_STATUS_UNKNOWN;
120 }
121
122 virtual bool HasPRead() const;
123 virtual size_t PRead(void *pBuffer, size_t nSize,
124 vsi_l_offset nOffset) const;
125
130 virtual void Interrupt()
131 {
132 }
133
134 // NOTE: when adding new methods, besides the "actual" implementations,
135 // also consider the VSICachedFile one.
136
137 virtual ~VSIVirtualHandle()
138 {
139 }
140};
141
142/************************************************************************/
143/* VSIVirtualHandleCloser */
144/************************************************************************/
145
148 struct VSIVirtualHandleCloser
149{
151 void operator()(VSIVirtualHandle *poHandle)
152 {
153 if (poHandle)
154 {
155 poHandle->Close();
156 delete poHandle;
157 }
158 }
159};
160
162 typedef std::unique_ptr<VSIVirtualHandle, VSIVirtualHandleCloser>
163 VSIVirtualHandleUniquePtr;
164
165/************************************************************************/
166/* VSIFilesystemHandler */
167/************************************************************************/
168
169#ifndef DOXYGEN_SKIP
170 class CPL_DLL VSIFilesystemHandler
171{
172
173 public:
174 virtual ~VSIFilesystemHandler()
175 {
176 }
177
178 VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess);
179
180 virtual VSIVirtualHandle *Open(const char *pszFilename,
181 const char *pszAccess, bool bSetError,
182 CSLConstList papszOptions) = 0;
183 virtual int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
184 int nFlags) = 0;
185
186 virtual int Unlink(const char *pszFilename)
187 {
188 (void)pszFilename;
189 errno = ENOENT;
190 return -1;
191 }
192
193 virtual int *UnlinkBatch(CSLConstList papszFiles);
194
195 virtual int Mkdir(const char *pszDirname, long nMode)
196 {
197 (void)pszDirname;
198 (void)nMode;
199 errno = ENOENT;
200 return -1;
201 }
202
203 virtual int Rmdir(const char *pszDirname)
204 {
205 (void)pszDirname;
206 errno = ENOENT;
207 return -1;
208 }
209
210 virtual int RmdirRecursive(const char *pszDirname);
211
212 char **ReadDir(const char *pszDirname)
213 {
214 return ReadDirEx(pszDirname, 0);
215 }
216
217 virtual char **ReadDirEx(const char * /*pszDirname*/, int /* nMaxFiles */)
218 {
219 return nullptr;
220 }
221
222 virtual char **SiblingFiles(const char * /*pszFilename*/)
223 {
224 return nullptr;
225 }
226
227 virtual int Rename(const char *oldpath, const char *newpath)
228 {
229 (void)oldpath;
230 (void)newpath;
231 errno = ENOENT;
232 return -1;
233 }
234
235 virtual int IsCaseSensitive(const char *pszFilename)
236 {
237 (void)pszFilename;
238 return TRUE;
239 }
240
241 virtual GIntBig GetDiskFreeSpace(const char * /* pszDirname */)
242 {
243 return -1;
244 }
245
246 virtual int SupportsSparseFiles(const char * /* pszPath */)
247 {
248 return FALSE;
249 }
250
251 virtual int HasOptimizedReadMultiRange(const char * /* pszPath */)
252 {
253 return FALSE;
254 }
255
256 virtual const char *GetActualURL(const char * /*pszFilename*/)
257 {
258 return nullptr;
259 }
260
261 virtual const char *GetOptions()
262 {
263 return nullptr;
264 }
265
266 virtual char *GetSignedURL(const char * /*pszFilename*/,
267 CSLConstList /* papszOptions */)
268 {
269 return nullptr;
270 }
271
272 virtual bool Sync(const char *pszSource, const char *pszTarget,
273 const char *const *papszOptions,
274 GDALProgressFunc pProgressFunc, void *pProgressData,
275 char ***ppapszOutputs);
276
277 virtual int CopyFile(const char *pszSource, const char *pszTarget,
278 VSILFILE *fpSource, vsi_l_offset nSourceSize,
279 const char *const *papszOptions,
280 GDALProgressFunc pProgressFunc, void *pProgressData);
281
282 virtual int
283 CopyFileRestartable(const char *pszSource, const char *pszTarget,
284 const char *pszInputPayload, char **ppszOutputPayload,
285 CSLConstList papszOptions,
286 GDALProgressFunc pProgressFunc, void *pProgressData);
287
288 virtual VSIDIR *OpenDir(const char *pszPath, int nRecurseDepth,
289 const char *const *papszOptions);
290
291 virtual char **GetFileMetadata(const char *pszFilename,
292 const char *pszDomain,
293 CSLConstList papszOptions);
294
295 virtual bool SetFileMetadata(const char *pszFilename,
296 CSLConstList papszMetadata,
297 const char *pszDomain,
298 CSLConstList papszOptions);
299
300 virtual bool
301 MultipartUploadGetCapabilities(int *pbNonSequentialUploadSupported,
302 int *pbParallelUploadSupported,
303 int *pbAbortSupported, size_t *pnMinPartSize,
304 size_t *pnMaxPartSize, int *pnMaxPartCount);
305
306 virtual char *MultipartUploadStart(const char *pszFilename,
307 CSLConstList papszOptions);
308
309 virtual char *MultipartUploadAddPart(const char *pszFilename,
310 const char *pszUploadId,
311 int nPartNumber,
312 vsi_l_offset nFileOffset,
313 const void *pData, size_t nDataLength,
314 CSLConstList papszOptions);
315
316 virtual bool
317 MultipartUploadEnd(const char *pszFilename, const char *pszUploadId,
318 size_t nPartIdsCount, const char *const *apszPartIds,
319 vsi_l_offset nTotalSize, CSLConstList papszOptions);
320
321 virtual bool MultipartUploadAbort(const char *pszFilename,
322 const char *pszUploadId,
323 CSLConstList papszOptions);
324
325 virtual bool AbortPendingUploads(const char * /*pszFilename*/)
326 {
327 return true;
328 }
329
330 virtual std::string
331 GetStreamingFilename(const std::string &osFilename) const
332 {
333 return osFilename;
334 }
335
336 virtual std::string
337 GetNonStreamingFilename(const std::string &osFilename) const
338 {
339 return osFilename;
340 }
341
349 virtual std::string
350 GetCanonicalFilename(const std::string &osFilename) const
351 {
352 return osFilename;
353 }
354
355 virtual bool IsLocal(const char * /* pszPath */)
356 {
357 return true;
358 }
359
360 virtual bool SupportsSequentialWrite(const char * /* pszPath */,
361 bool /* bAllowLocalTempFile */)
362 {
363 return true;
364 }
365
366 virtual bool SupportsRandomWrite(const char * /* pszPath */,
367 bool /* bAllowLocalTempFile */)
368 {
369 return true;
370 }
371
372 virtual bool SupportsRead(const char * /* pszPath */)
373 {
374 return true;
375 }
376
377 virtual VSIFilesystemHandler *Duplicate(const char * /* pszPrefix */)
378 {
379 CPLError(CE_Failure, CPLE_NotSupported,
380 "Duplicate() not supported on this file system");
381 return nullptr;
382 }
383
390 virtual const char *GetDirectorySeparator(CPL_UNUSED const char *pszPath)
391 {
392 return "/";
393 }
394};
395#endif /* #ifndef DOXYGEN_SKIP */
396
397/************************************************************************/
398/* VSIFileManager */
399/************************************************************************/
400
401#ifndef DOXYGEN_SKIP
402 class CPL_DLL VSIFileManager
403{
404 private:
405 VSIFilesystemHandler *poDefaultHandler = nullptr;
406 std::map<std::string, VSIFilesystemHandler *> oHandlers{};
407
408 VSIFileManager();
409
410 static VSIFileManager *Get();
411
412 CPL_DISALLOW_COPY_ASSIGN(VSIFileManager)
413
414 public:
415 ~VSIFileManager();
416
417 static VSIFilesystemHandler *GetHandler(const char *);
418 static void InstallHandler(const std::string &osPrefix,
419 VSIFilesystemHandler *);
420 static void RemoveHandler(const std::string &osPrefix);
421
422 static char **GetPrefixes();
423};
424#endif /* #ifndef DOXYGEN_SKIP */
425
426/************************************************************************/
427/* ==================================================================== */
428/* VSIArchiveFilesystemHandler */
429/* ==================================================================== */
430/************************************************************************/
431
432#ifndef DOXYGEN_SKIP
433
434 class VSIArchiveEntryFileOffset
435{
436 public:
437 virtual ~VSIArchiveEntryFileOffset();
438};
439
440 typedef struct
441{
442 char *fileName;
443 vsi_l_offset uncompressed_size;
444 VSIArchiveEntryFileOffset *file_pos;
445 int bIsDir;
446 GIntBig nModifiedTime;
447} VSIArchiveEntry;
448
449 class VSIArchiveContent
450{
451 public:
452 time_t mTime = 0;
453 vsi_l_offset nFileSize = 0;
454 int nEntries = 0;
455 VSIArchiveEntry *entries = nullptr;
456
457 ~VSIArchiveContent();
458};
459
460 class VSIArchiveReader
461{
462 public:
463 virtual ~VSIArchiveReader();
464
465 virtual int GotoFirstFile() = 0;
466 virtual int GotoNextFile() = 0;
467 virtual VSIArchiveEntryFileOffset *GetFileOffset() = 0;
468 virtual GUIntBig GetFileSize() = 0;
469 virtual CPLString GetFileName() = 0;
470 virtual GIntBig GetModifiedTime() = 0;
471 virtual int GotoFileOffset(VSIArchiveEntryFileOffset *pOffset) = 0;
472};
473
474 class VSIArchiveFilesystemHandler : public VSIFilesystemHandler
475{
476 CPL_DISALLOW_COPY_ASSIGN(VSIArchiveFilesystemHandler)
477
478 protected:
479 CPLMutex *hMutex = nullptr;
480 /* We use a cache that contains the list of files contained in a VSIArchive
481 * file as */
482 /* unarchive.c is quite inefficient in listing them. This speeds up access
483 * to VSIArchive files */
484 /* containing ~1000 files like a CADRG product */
485 std::map<CPLString, VSIArchiveContent *> oFileList{};
486
487 virtual const char *GetPrefix() = 0;
488 virtual std::vector<CPLString> GetExtensions() = 0;
489 virtual VSIArchiveReader *CreateReader(const char *pszArchiveFileName) = 0;
490
491 public:
492 VSIArchiveFilesystemHandler();
493 virtual ~VSIArchiveFilesystemHandler();
494
495 int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
496 int nFlags) override;
497 int Unlink(const char *pszFilename) override;
498 int Rename(const char *oldpath, const char *newpath) override;
499 int Mkdir(const char *pszDirname, long nMode) override;
500 int Rmdir(const char *pszDirname) override;
501 char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
502
503 virtual const VSIArchiveContent *
504 GetContentOfArchive(const char *archiveFilename,
505 VSIArchiveReader *poReader = nullptr);
506 virtual char *SplitFilename(const char *pszFilename,
507 CPLString &osFileInArchive,
508 int bCheckMainFileExists);
509 virtual VSIArchiveReader *OpenArchiveFile(const char *archiveFilename,
510 const char *fileInArchiveName);
511 virtual int FindFileInArchive(const char *archiveFilename,
512 const char *fileInArchiveName,
513 const VSIArchiveEntry **archiveEntry);
514
515 virtual bool IsLocal(const char *pszPath) override;
516
517 virtual bool
518 SupportsSequentialWrite(const char * /* pszPath */,
519 bool /* bAllowLocalTempFile */) override
520 {
521 return false;
522 }
523
524 virtual bool SupportsRandomWrite(const char * /* pszPath */,
525 bool /* bAllowLocalTempFile */) override
526 {
527 return false;
528 }
529};
530
531/************************************************************************/
532/* VSIDIR */
533/************************************************************************/
534
535 struct CPL_DLL VSIDIR
536{
537 VSIDIR() = default;
538 virtual ~VSIDIR();
539
540 virtual const VSIDIREntry *NextDirEntry() = 0;
541
542 private:
543 VSIDIR(const VSIDIR &) = delete;
544 VSIDIR &operator=(const VSIDIR &) = delete;
545};
546
547#endif /* #ifndef DOXYGEN_SKIP */
548
549 VSIVirtualHandle CPL_DLL *
550VSICreateBufferedReaderHandle(VSIVirtualHandle *poBaseHandle);
551 VSIVirtualHandle *
552VSICreateBufferedReaderHandle(VSIVirtualHandle *poBaseHandle,
553 const GByte *pabyBeginningContent,
554 vsi_l_offset nCheatFileSize);
555 constexpr int VSI_CACHED_DEFAULT_CHUNK_SIZE = 32768;
556 VSIVirtualHandle CPL_DLL *
557VSICreateCachedFile(VSIVirtualHandle *poBaseHandle,
558 size_t nChunkSize = VSI_CACHED_DEFAULT_CHUNK_SIZE,
559 size_t nCacheSize = 0);
560
561 const int CPL_DEFLATE_TYPE_GZIP = 0;
562 const int CPL_DEFLATE_TYPE_ZLIB = 1;
563 const int CPL_DEFLATE_TYPE_RAW_DEFLATE = 2;
564 VSIVirtualHandle CPL_DLL *VSICreateGZipWritable(VSIVirtualHandle *poBaseHandle,
565 int nDeflateType,
566 int bAutoCloseBaseHandle);
567
568 VSIVirtualHandle *VSICreateGZipWritable(VSIVirtualHandle *poBaseHandle,
569 int nDeflateType,
570 bool bAutoCloseBaseHandle, int nThreads,
571 size_t nChunkSize,
572 size_t nSOZIPIndexEltSize,
573 std::vector<uint8_t> *panSOZIPIndex);
574
575 VSIVirtualHandle *
576VSICreateUploadOnCloseFile(VSIVirtualHandleUniquePtr &&poWritableHandle,
577 VSIVirtualHandleUniquePtr &&poTmpFile,
578 const std::string &osTmpFilename);
579
580#endif /* ndef CPL_VSI_VIRTUAL_H_INCLUDED */
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:307
CPLE_NotSupported
#define CPLE_NotSupported
Not supported.
Definition: cpl_error.h:94
CPLError
void CPLError(CPLErr eErrClass, CPLErrorNum err_no, const char *fmt,...)
Report an error.
Definition: cpl_error.cpp:314
GUIntBig
unsigned long long GUIntBig
Large unsigned integer type (generally 64-bit unsigned integer type).
Definition: cpl_port.h:202
CPL_UNUSED
#define CPL_UNUSED
Qualifier for an argument that is unused.
Definition: cpl_port.h:973
CPL_FORMAT_STRING
#define CPL_FORMAT_STRING(arg)
Macro into which to wrap the format argument of a printf-like function.
Definition: cpl_port.h:954
CPL_PRINT_FUNC_FORMAT
#define CPL_PRINT_FUNC_FORMAT(format_idx, arg_idx)
Tag a function to have printf() formatting.
Definition: cpl_port.h:938
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1030
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1179
GByte
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:169
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:199
cpl_string.h
Various convenience functions for working with strings and string lists.
cpl_vsi.h
Standard C Covers.
VSIStatBufL
#define VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:189
VSIRangeStatus
VSIRangeStatus
Range status.
Definition: cpl_vsi.h:168
VSI_RANGE_STATUS_UNKNOWN
@ VSI_RANGE_STATUS_UNKNOWN
Unknown.
Definition: cpl_vsi.h:169
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:388
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:130
VSIDIREntry
Directory entry.
Definition: cpl_vsi.h:400
VSIVirtualHandleCloser
Helper close to use with a std:unique_ptr<VSIVirtualHandle>, such as VSIVirtualHandleUniquePtr.
Definition: cpl_vsi_virtual.h:149
VSIVirtualHandleCloser::operator()
void operator()(VSIVirtualHandle *poHandle)
Operator () that closes and deletes the file handle.
Definition: cpl_vsi_virtual.h:151
VSIVirtualHandle
Virtual file handle.
Definition: cpl_vsi_virtual.h:47
VSIVirtualHandle::Close
virtual int Close()=0
Close file.
VSIVirtualHandle::GetRangeStatus
virtual VSIRangeStatus GetRangeStatus(vsi_l_offset nOffset, vsi_l_offset nLength)
Return if a given file range contains data or holes filled with zeroes.
Definition: cpl_vsi_virtual.h:116
VSIVirtualHandle::Tell
virtual vsi_l_offset Tell()=0
Tell current file offset.
VSIVirtualHandle::Seek
virtual int Seek(vsi_l_offset nOffset, int nWhence)=0
Seek to requested offset.
VSIVirtualHandle::Read
virtual size_t Read(void *pBuffer, size_t nSize, size_t nCount)=0
Read bytes from file.
VSIVirtualHandle::GetNativeFileDescriptor
virtual void * GetNativeFileDescriptor()
Returns the "native" file descriptor for the virtual handle.
Definition: cpl_vsi_virtual.h:111
VSIVirtualHandle::GetAdviseReadTotalBytesLimit
virtual size_t GetAdviseReadTotalBytesLimit() const
Return the total maximum number of bytes that AdviseRead() can handle at once.
Definition: cpl_vsi_virtual.h:86
VSIVirtualHandle::AdviseRead
virtual void AdviseRead(int nRanges, const vsi_l_offset *panOffsets, const size_t *panSizes)
This method is called when code plans to access soon one or several ranges in a file.
Definition: cpl_vsi_virtual.h:68
VSIVirtualHandle::ReadMultiRange
virtual int ReadMultiRange(int nRanges, void **ppData, const vsi_l_offset *panOffsets, const size_t *panSizes)
Read several ranges of bytes from file.
VSIVirtualHandle::Write
virtual size_t Write(const void *pBuffer, size_t nSize, size_t nCount)=0
Write bytes to file.
VSIVirtualHandle::Interrupt
virtual void Interrupt()
Ask current operations to be interrupted.
Definition: cpl_vsi_virtual.h:130
VSIVirtualHandle::Truncate
virtual int Truncate(vsi_l_offset nNewSize)
Truncate/expand the file to the specified size.