GDAL

cpl_vsi_virtual.h
1 /******************************************************************************
2 * $Id$
3 *
4 * Project: VSI Virtual File System
5 * Purpose: Declarations for classes related to the virtual filesystem.
6 * These would only be normally required by applications implementing
7 * their own virtual file system classes which should be rare.
8 * The class interface may be fragile through versions.
9 * Author: Frank Warmerdam, warmerdam@pobox.com
10 *
11 ******************************************************************************
12 * Copyright (c) 2005, Frank Warmerdam <warmerdam@pobox.com>
13 * Copyright (c) 2010-2014, Even Rouault <even dot rouault at spatialys.com>
14 *
15 * Permission is hereby granted, free of charge, to any person obtaining a
16 * copy of this software and associated documentation files (the "Software"),
17 * to deal in the Software without restriction, including without limitation
18 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
19 * and/or sell copies of the Software, and to permit persons to whom the
20 * Software is furnished to do so, subject to the following conditions:
21 *
22 * The above copyright notice and this permission notice shall be included
23 * in all copies or substantial portions of the Software.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
26 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
28 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
31 * DEALINGS IN THE SOFTWARE.
32 ****************************************************************************/
33
34 #ifndef CPL_VSI_VIRTUAL_H_INCLUDED
35 #define CPL_VSI_VIRTUAL_H_INCLUDED
36
37 #include "cpl_vsi.h"
38 #include "cpl_vsi_error.h"
39 #include "cpl_string.h"
40 #include "cpl_multiproc.h"
41
42 #include <map>
43 #include <memory>
44 #include <vector>
45 #include <string>
46
47 // To avoid aliasing to GetDiskFreeSpace to GetDiskFreeSpaceA on Windows
48 #ifdef GetDiskFreeSpace
49 #undef GetDiskFreeSpace
50 #endif
51
52 // To avoid aliasing to CopyFile to CopyFileA on Windows
53 #ifdef CopyFile
54 #undef CopyFile
55 #endif
56
57 /************************************************************************/
58 /* VSIVirtualHandle */
59 /************************************************************************/
60
62 struct CPL_DLL VSIVirtualHandle
63 {
64 public:
65 virtual int Seek(vsi_l_offset nOffset, int nWhence) = 0;
66 virtual vsi_l_offset Tell() = 0;
67 virtual size_t Read(void *pBuffer, size_t nSize, size_t nCount) = 0;
68 virtual int ReadMultiRange(int nRanges, void **ppData,
69 const vsi_l_offset *panOffsets,
70 const size_t *panSizes);
71
84 virtual void AdviseRead(CPL_UNUSED int nRanges,
85 CPL_UNUSED const vsi_l_offset *panOffsets,
86 CPL_UNUSED const size_t *panSizes)
87 {
88 }
89
90 virtual size_t Write(const void *pBuffer, size_t nSize, size_t nCount) = 0;
91 virtual int Eof() = 0;
92 virtual int Flush()
93 {
94 return 0;
95 }
96 virtual int Close() = 0;
97 // Base implementation that only supports file extension.
98 virtual int Truncate(vsi_l_offset nNewSize);
99 virtual void *GetNativeFileDescriptor()
100 {
101 return nullptr;
102 }
103 virtual VSIRangeStatus GetRangeStatus(CPL_UNUSED vsi_l_offset nOffset,
104 CPL_UNUSED vsi_l_offset nLength)
105 {
106 return VSI_RANGE_STATUS_UNKNOWN;
107 }
108 virtual bool HasPRead() const;
109 virtual size_t PRead(void *pBuffer, size_t nSize,
110 vsi_l_offset nOffset) const;
111
112 // NOTE: when adding new methods, besides the "actual" implementations,
113 // also consider the VSICachedFile one.
114
115 virtual ~VSIVirtualHandle()
116 {
117 }
118 };
119
120 /************************************************************************/
121 /* VSIVirtualHandleCloser */
122 /************************************************************************/
123
126 struct VSIVirtualHandleCloser
127 {
129 void operator()(VSIVirtualHandle *poHandle)
130 {
131 if (poHandle)
132 {
133 poHandle->Close();
134 delete poHandle;
135 }
136 }
137 };
138
140 typedef std::unique_ptr<VSIVirtualHandle, VSIVirtualHandleCloser>
141 VSIVirtualHandleUniquePtr;
142
143 /************************************************************************/
144 /* VSIFilesystemHandler */
145 /************************************************************************/
146
147 #ifndef DOXYGEN_SKIP
148 class CPL_DLL VSIFilesystemHandler
149 {
150
151 public:
152 virtual ~VSIFilesystemHandler()
153 {
154 }
155
156 VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess);
157
158 virtual VSIVirtualHandle *Open(const char *pszFilename,
159 const char *pszAccess, bool bSetError,
160 CSLConstList papszOptions) = 0;
161 virtual int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
162 int nFlags) = 0;
163 virtual int Unlink(const char *pszFilename)
164 {
165 (void)pszFilename;
166 errno = ENOENT;
167 return -1;
168 }
169 virtual int *UnlinkBatch(CSLConstList papszFiles);
170 virtual int Mkdir(const char *pszDirname, long nMode)
171 {
172 (void)pszDirname;
173 (void)nMode;
174 errno = ENOENT;
175 return -1;
176 }
177 virtual int Rmdir(const char *pszDirname)
178 {
179 (void)pszDirname;
180 errno = ENOENT;
181 return -1;
182 }
183 virtual int RmdirRecursive(const char *pszDirname);
184 char **ReadDir(const char *pszDirname)
185 {
186 return ReadDirEx(pszDirname, 0);
187 }
188 virtual char **ReadDirEx(const char * /*pszDirname*/, int /* nMaxFiles */)
189 {
190 return nullptr;
191 }
192 virtual char **SiblingFiles(const char * /*pszFilename*/)
193 {
194 return nullptr;
195 }
196 virtual int Rename(const char *oldpath, const char *newpath)
197 {
198 (void)oldpath;
199 (void)newpath;
200 errno = ENOENT;
201 return -1;
202 }
203 virtual int IsCaseSensitive(const char *pszFilename)
204 {
205 (void)pszFilename;
206 return TRUE;
207 }
208 virtual GIntBig GetDiskFreeSpace(const char * /* pszDirname */)
209 {
210 return -1;
211 }
212 virtual int SupportsSparseFiles(const char * /* pszPath */)
213 {
214 return FALSE;
215 }
216 virtual int HasOptimizedReadMultiRange(const char * /* pszPath */)
217 {
218 return FALSE;
219 }
220 virtual const char *GetActualURL(const char * /*pszFilename*/)
221 {
222 return nullptr;
223 }
224 virtual const char *GetOptions()
225 {
226 return nullptr;
227 }
228 virtual char *GetSignedURL(const char * /*pszFilename*/,
229 CSLConstList /* papszOptions */)
230 {
231 return nullptr;
232 }
233 virtual bool Sync(const char *pszSource, const char *pszTarget,
234 const char *const *papszOptions,
235 GDALProgressFunc pProgressFunc, void *pProgressData,
236 char ***ppapszOutputs);
237
238 virtual int CopyFile(const char *pszSource, const char *pszTarget,
239 VSILFILE *fpSource, vsi_l_offset nSourceSize,
240 const char *const *papszOptions,
241 GDALProgressFunc pProgressFunc, void *pProgressData);
242
243 virtual VSIDIR *OpenDir(const char *pszPath, int nRecurseDepth,
244 const char *const *papszOptions);
245
246 virtual char **GetFileMetadata(const char *pszFilename,
247 const char *pszDomain,
248 CSLConstList papszOptions);
249
250 virtual bool SetFileMetadata(const char *pszFilename,
251 CSLConstList papszMetadata,
252 const char *pszDomain,
253 CSLConstList papszOptions);
254
255 virtual bool AbortPendingUploads(const char * /*pszFilename*/)
256 {
257 return true;
258 }
259
260 virtual std::string
261 GetStreamingFilename(const std::string &osFilename) const
262 {
263 return osFilename;
264 }
265
273 virtual std::string
274 GetCanonicalFilename(const std::string &osFilename) const
275 {
276 return osFilename;
277 }
278
279 virtual bool IsLocal(const char * /* pszPath */)
280 {
281 return true;
282 }
283 virtual bool SupportsSequentialWrite(const char * /* pszPath */,
284 bool /* bAllowLocalTempFile */)
285 {
286 return true;
287 }
288 virtual bool SupportsRandomWrite(const char * /* pszPath */,
289 bool /* bAllowLocalTempFile */)
290 {
291 return true;
292 }
293 virtual bool SupportsRead(const char * /* pszPath */)
294 {
295 return true;
296 }
297
298 virtual VSIFilesystemHandler *Duplicate(const char * /* pszPrefix */)
299 {
300 CPLError(CE_Failure, CPLE_NotSupported,
301 "Duplicate() not supported on this file system");
302 return nullptr;
303 }
304 };
305 #endif /* #ifndef DOXYGEN_SKIP */
306
307 /************************************************************************/
308 /* VSIFileManager */
309 /************************************************************************/
310
311 #ifndef DOXYGEN_SKIP
312 class CPL_DLL VSIFileManager
313 {
314 private:
315 VSIFilesystemHandler *poDefaultHandler = nullptr;
316 std::map<std::string, VSIFilesystemHandler *> oHandlers{};
317
318 VSIFileManager();
319
320 static VSIFileManager *Get();
321
322 CPL_DISALLOW_COPY_ASSIGN(VSIFileManager)
323
324 public:
325 ~VSIFileManager();
326
327 static VSIFilesystemHandler *GetHandler(const char *);
328 static void InstallHandler(const std::string &osPrefix,
329 VSIFilesystemHandler *);
330 /* RemoveHandler is never defined. */
331 /* static void RemoveHandler( const std::string& osPrefix ); */
332
333 static char **GetPrefixes();
334 };
335 #endif /* #ifndef DOXYGEN_SKIP */
336
337 /************************************************************************/
338 /* ==================================================================== */
339 /* VSIArchiveFilesystemHandler */
340 /* ==================================================================== */
341 /************************************************************************/
342
343 #ifndef DOXYGEN_SKIP
344
345 class VSIArchiveEntryFileOffset
346 {
347 public:
348 virtual ~VSIArchiveEntryFileOffset();
349 };
350
351 typedef struct
352 {
353 char *fileName;
354 vsi_l_offset uncompressed_size;
355 VSIArchiveEntryFileOffset *file_pos;
356 int bIsDir;
357 GIntBig nModifiedTime;
358 } VSIArchiveEntry;
359
360 class VSIArchiveContent
361 {
362 public:
363 time_t mTime = 0;
364 vsi_l_offset nFileSize = 0;
365 int nEntries = 0;
366 VSIArchiveEntry *entries = nullptr;
367
368 ~VSIArchiveContent();
369 };
370
371 class VSIArchiveReader
372 {
373 public:
374 virtual ~VSIArchiveReader();
375
376 virtual int GotoFirstFile() = 0;
377 virtual int GotoNextFile() = 0;
378 virtual VSIArchiveEntryFileOffset *GetFileOffset() = 0;
379 virtual GUIntBig GetFileSize() = 0;
380 virtual CPLString GetFileName() = 0;
381 virtual GIntBig GetModifiedTime() = 0;
382 virtual int GotoFileOffset(VSIArchiveEntryFileOffset *pOffset) = 0;
383 };
384
385 class VSIArchiveFilesystemHandler : public VSIFilesystemHandler
386 {
387 CPL_DISALLOW_COPY_ASSIGN(VSIArchiveFilesystemHandler)
388
389 protected:
390 CPLMutex *hMutex = nullptr;
391 /* We use a cache that contains the list of files contained in a VSIArchive
392 * file as */
393 /* unarchive.c is quite inefficient in listing them. This speeds up access
394 * to VSIArchive files */
395 /* containing ~1000 files like a CADRG product */
396 std::map<CPLString, VSIArchiveContent *> oFileList{};
397
398 virtual const char *GetPrefix() = 0;
399 virtual std::vector<CPLString> GetExtensions() = 0;
400 virtual VSIArchiveReader *CreateReader(const char *pszArchiveFileName) = 0;
401
402 public:
403 VSIArchiveFilesystemHandler();
404 virtual ~VSIArchiveFilesystemHandler();
405
406 int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
407 int nFlags) override;
408 int Unlink(const char *pszFilename) override;
409 int Rename(const char *oldpath, const char *newpath) override;
410 int Mkdir(const char *pszDirname, long nMode) override;
411 int Rmdir(const char *pszDirname) override;
412 char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
413
414 virtual const VSIArchiveContent *
415 GetContentOfArchive(const char *archiveFilename,
416 VSIArchiveReader *poReader = nullptr);
417 virtual char *SplitFilename(const char *pszFilename,
418 CPLString &osFileInArchive,
419 int bCheckMainFileExists);
420 virtual VSIArchiveReader *OpenArchiveFile(const char *archiveFilename,
421 const char *fileInArchiveName);
422 virtual int FindFileInArchive(const char *archiveFilename,
423 const char *fileInArchiveName,
424 const VSIArchiveEntry **archiveEntry);
425
426 virtual bool IsLocal(const char *pszPath) override;
427 virtual bool
428 SupportsSequentialWrite(const char * /* pszPath */,
429 bool /* bAllowLocalTempFile */) override
430 {
431 return false;
432 }
433 virtual bool SupportsRandomWrite(const char * /* pszPath */,
434 bool /* bAllowLocalTempFile */) override
435 {
436 return false;
437 }
438 };
439
440 /************************************************************************/
441 /* VSIDIR */
442 /************************************************************************/
443
444 struct CPL_DLL VSIDIR
445 {
446 VSIDIR() = default;
447 virtual ~VSIDIR();
448
449 virtual const VSIDIREntry *NextDirEntry() = 0;
450
451 private:
452 VSIDIR(const VSIDIR &) = delete;
453 VSIDIR &operator=(const VSIDIR &) = delete;
454 };
455
456 #endif /* #ifndef DOXYGEN_SKIP */
457
458 VSIVirtualHandle CPL_DLL *
459 VSICreateBufferedReaderHandle(VSIVirtualHandle *poBaseHandle);
460 VSIVirtualHandle *
461 VSICreateBufferedReaderHandle(VSIVirtualHandle *poBaseHandle,
462 const GByte *pabyBeginningContent,
463 vsi_l_offset nCheatFileSize);
464 constexpr int VSI_CACHED_DEFAULT_CHUNK_SIZE = 32768;
465 VSIVirtualHandle CPL_DLL *
466 VSICreateCachedFile(VSIVirtualHandle *poBaseHandle,
467 size_t nChunkSize = VSI_CACHED_DEFAULT_CHUNK_SIZE,
468 size_t nCacheSize = 0);
469
470 const int CPL_DEFLATE_TYPE_GZIP = 0;
471 const int CPL_DEFLATE_TYPE_ZLIB = 1;
472 const int CPL_DEFLATE_TYPE_RAW_DEFLATE = 2;
473 VSIVirtualHandle CPL_DLL *VSICreateGZipWritable(VSIVirtualHandle *poBaseHandle,
474 int nDeflateType,
475 int bAutoCloseBaseHandle);
476
477 VSIVirtualHandle *VSICreateGZipWritable(VSIVirtualHandle *poBaseHandle,
478 int nDeflateType,
479 bool bAutoCloseBaseHandle, int nThreads,
480 size_t nChunkSize,
481 size_t nSOZIPIndexEltSize,
482 std::vector<uint8_t> *panSOZIPIndex);
483
484 VSIVirtualHandle *
485 VSICreateUploadOnCloseFile(VSIVirtualHandleUniquePtr &&poWritableHandle,
486 VSIVirtualHandleUniquePtr &&poTmpFile,
487 const std::string &osTmpFilename);
488
489 #endif /* ndef CPL_VSI_VIRTUAL_H_INCLUDED */
GByte
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:196
VSIVirtualHandleCloser::operator()
void operator()(VSIVirtualHandle *poHandle)
Operator () that closes and deletes the file handle.
Definition: cpl_vsi_virtual.h:129
cpl_vsi.h
VSIVirtualHandle::Close
virtual int Close()=0
Close file.
VSIRangeStatus
VSIRangeStatus
Range status.
Definition: cpl_vsi.h:181
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:311
VSI_RANGE_STATUS_UNKNOWN
@ VSI_RANGE_STATUS_UNKNOWN
Unknown.
Definition: cpl_vsi.h:183
VSIVirtualHandleCloser
Helper close to use with a std:unique_ptr<VSIVirtualHandle>, such as VSIVirtualHandleUniquePtr.
Definition: cpl_vsi_virtual.h:126
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1178
GUIntBig
unsigned long long GUIntBig
Large unsigned integer type (generally 64-bit unsigned integer type).
Definition: cpl_port.h:229
VSIDIREntry
Directory entry.
Definition: cpl_vsi.h:394
cpl_string.h
VSIVirtualHandle
Virtual file handle.
Definition: cpl_vsi_virtual.h:62
CPLError
void CPLError(CPLErr eErrClass, CPLErrorNum err_no, const char *fmt,...)
Report an error.
Definition: cpl_error.cpp:323
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:146
CPLE_NotSupported
#define CPLE_NotSupported
Not supported.
Definition: cpl_error.h:110
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:226
CPL_UNUSED
#define CPL_UNUSED
Qualifier for an argument that is unused.
Definition: cpl_port.h:985
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:384
VSIStatBufL
#define VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:203
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1042