GDAL

cpl_aws.h
1 /**********************************************************************
2 * $Id$
3 *
4 * Name: cpl_aws.h
5 * Project: CPL - Common Portability Library
6 * Purpose: Amazon Web Services routines
7 * Author: Even Rouault <even.rouault at spatialys.com>
8 *
9 **********************************************************************
10 * Copyright (c) 2015, Even Rouault <even.rouault at spatialys.com>
11 *
12 * Permission is hereby granted, free of charge, to any person obtaining a
13 * copy of this software and associated documentation files (the "Software"),
14 * to deal in the Software without restriction, including without limitation
15 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
16 * and/or sell copies of the Software, and to permit persons to whom the
17 * Software is furnished to do so, subject to the following conditions:
18 *
19 * The above copyright notice and this permission notice shall be included
20 * in all copies or substantial portions of the Software.
21 *
22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
23 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
25 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
28 * DEALINGS IN THE SOFTWARE.
29 ****************************************************************************/
30
31 #ifndef CPL_AWS_INCLUDED_H
32 #define CPL_AWS_INCLUDED_H
33
34 #ifndef DOXYGEN_SKIP
35
36 #ifdef HAVE_CURL
37
38 #include <cstddef>
39 #include <mutex>
40
41 #include "cpl_string.h"
42
43 #include <curl/curl.h>
44 #include <map>
45
46 CPLString CPLGetLowerCaseHexSHA256( const void *pabyData, size_t nBytes );
47 CPLString CPLGetLowerCaseHexSHA256( const CPLString& osStr );
48
49 CPLString CPLGetAWS_SIGN4_Timestamp(GIntBig timestamp);
50
51 CPLString CPLAWSURLEncode(const CPLString& osURL, bool bEncodeSlash = true);
52
53 CPLString CPLAWSGetHeaderVal(const struct curl_slist* psExistingHeaders,
54 const char* pszKey);
55
56 CPLString
57 CPLGetAWS_SIGN4_Signature( const CPLString& osSecretAccessKey,
58 const CPLString& osAccessToken,
59 const CPLString& osRegion,
60 const CPLString& osRequestPayer,
61 const CPLString& osService,
62 const CPLString& osVerb,
63 const struct curl_slist* psExistingHeaders,
64 const CPLString& osHost,
65 const CPLString& osCanonicalURI,
66 const CPLString& osCanonicalQueryString,
67 const CPLString& osXAMZContentSHA256,
68 bool bAddHeaderAMZContentSHA256,
69 const CPLString& osTimestamp,
70 CPLString& osSignedHeaders );
71
72 CPLString CPLGetAWS_SIGN4_Authorization(const CPLString& osSecretAccessKey,
73 const CPLString& osAccessKeyId,
74 const CPLString& osAccessToken,
75 const CPLString& osRegion,
76 const CPLString& osRequestPayer,
77 const CPLString& osService,
78 const CPLString& osVerb,
79 const struct curl_slist* psExistingHeaders,
80 const CPLString& osHost,
81 const CPLString& osCanonicalURI,
82 const CPLString& osCanonicalQueryString,
83 const CPLString& osXAMZContentSHA256,
84 bool bAddHeaderAMZContentSHA256,
85 const CPLString& osTimestamp);
86
87 class IVSIS3LikeHandleHelper
88 {
89 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandleHelper)
90
91 protected:
92 std::map<CPLString, CPLString> m_oMapQueryParameters{};
93
94 virtual void RebuildURL() = 0;
95 CPLString GetQueryString(bool bAddEmptyValueAfterEqual) const;
96
97 public:
98 IVSIS3LikeHandleHelper() = default;
99 virtual ~IVSIS3LikeHandleHelper() = default;
100
101 void ResetQueryParameters();
102 void AddQueryParameter(const CPLString& osKey, const CPLString& osValue);
103
104 virtual struct curl_slist* GetCurlHeaders(const CPLString& osVerb,
105 const struct curl_slist* psExistingHeaders,
106 const void *pabyDataContent = nullptr,
107 size_t nBytesContent = 0) const = 0;
108
109 virtual bool AllowAutomaticRedirection() { return true; }
110 virtual bool CanRestartOnError(const char*, const char* /* pszHeaders*/,
111 bool /*bSetError*/, bool* /*pbUpdateMap*/ = nullptr) { return false;}
112
113 virtual const CPLString& GetURL() const = 0;
114 CPLString GetURLNoKVP() const;
115
116 virtual CPLString GetCopySourceHeader() const { return std::string(); }
117 virtual const char* GetMetadataDirectiveREPLACE() const { return ""; }
118
119 static bool GetBucketAndObjectKey(const char* pszURI,
120 const char* pszFSPrefix,
121 bool bAllowNoObject,
122 CPLString &osBucketOut,
123 CPLString &osObjectKeyOut);
124
125 static CPLString BuildCanonicalizedHeaders(
126 std::map<CPLString, CPLString>& oSortedMapHeaders,
127 const struct curl_slist* psExistingHeaders,
128 const char* pszHeaderPrefix);
129
130 static CPLString GetRFC822DateTime();
131 };
132
133 enum class AWSCredentialsSource
134 {
135 REGULAR, // credentials from env variables or ~/.aws/crediential
136 EC2, // credentials from EC2 private networking
137 WEB_IDENTITY, // credentials from Web Identity Token
138 // See https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html
139 ASSUMED_ROLE // credentials from an STS assumed role
140 // See https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-cli.html
141 // and https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html
142 };
143
144 class VSIS3HandleHelper final: public IVSIS3LikeHandleHelper
145 {
146 CPL_DISALLOW_COPY_ASSIGN(VSIS3HandleHelper)
147
148 CPLString m_osURL{};
149 mutable CPLString m_osSecretAccessKey{};
150 mutable CPLString m_osAccessKeyId{};
151 mutable CPLString m_osSessionToken{};
152 CPLString m_osEndpoint{};
153 CPLString m_osRegion{};
154 CPLString m_osRequestPayer{};
155 CPLString m_osBucket{};
156 CPLString m_osObjectKey{};
157 bool m_bUseHTTPS = false;
158 bool m_bUseVirtualHosting = false;
159 AWSCredentialsSource m_eCredentialsSource = AWSCredentialsSource::REGULAR;
160
161 void RebuildURL() override;
162
163 static bool GetOrRefreshTemporaryCredentialsForRole(bool bForceRefresh,
164 CPLString& osSecretAccessKey,
165 CPLString& osAccessKeyId,
166 CPLString& osSessionToken,
167 CPLString& osRegion);
168
169 static bool GetConfigurationFromAssumeRoleWithWebIdentity(bool bForceRefresh,
170 const std::string& osPathForOption,
171 const std::string& osRoleArnIn,
172 const std::string& osWebIdentityTokenFileIn,
173 CPLString& osSecretAccessKey,
174 CPLString& osAccessKeyId,
175 CPLString& osSessionToken);
176
177 static bool GetConfigurationFromEC2(bool bForceRefresh,
178 const std::string& osPathForOption,
179 CPLString& osSecretAccessKey,
180 CPLString& osAccessKeyId,
181 CPLString& osSessionToken);
182
183 static bool GetConfigurationFromAWSConfigFiles(
184 const std::string& osPathForOption,
185 const char* pszProfile,
186 CPLString& osSecretAccessKey,
187 CPLString& osAccessKeyId,
188 CPLString& osSessionToken,
189 CPLString& osRegion,
190 CPLString& osCredentials,
191 CPLString& osRoleArn,
192 CPLString& osSourceProfile,
193 CPLString& osExternalId,
194 CPLString& osMFASerial,
195 CPLString& osRoleSessionName,
196 CPLString& osWebIdentityTokenFile);
197
198 static bool GetConfiguration(const std::string& osPathForOption,
199 CSLConstList papszOptions,
200 CPLString& osSecretAccessKey,
201 CPLString& osAccessKeyId,
202 CPLString& osSessionToken,
203 CPLString& osRegion,
204 AWSCredentialsSource& eCredentialsSource);
205
206 void RefreshCredentials(const std::string& osPathForOption,
207 bool bForceRefresh) const;
208
209 protected:
210
211 public:
212 VSIS3HandleHelper(const CPLString& osSecretAccessKey,
213 const CPLString& osAccessKeyId,
214 const CPLString& osSessionToken,
215 const CPLString& osEndpoint,
216 const CPLString& osRegion,
217 const CPLString& osRequestPayer,
218 const CPLString& osBucket,
219 const CPLString& osObjectKey,
220 bool bUseHTTPS, bool bUseVirtualHosting,
221 AWSCredentialsSource eCredentialsSource);
222 ~VSIS3HandleHelper();
223
224 static VSIS3HandleHelper* BuildFromURI(const char* pszURI,
225 const char* pszFSPrefix,
226 bool bAllowNoObject,
227 CSLConstList papszOptions = nullptr);
228 static CPLString BuildURL(const CPLString& osEndpoint,
229 const CPLString& osBucket,
230 const CPLString& osObjectKey,
231 bool bUseHTTPS, bool bUseVirtualHosting);
232
233 struct curl_slist* GetCurlHeaders(
234 const CPLString& osVerb,
235 const struct curl_slist* psExistingHeaders,
236 const void *pabyDataContent = nullptr,
237 size_t nBytesContent = 0) const override;
238
239 bool AllowAutomaticRedirection() override { return false; }
240 bool CanRestartOnError(const char*, const char* pszHeaders,
241 bool bSetError,
242 bool* pbUpdateMap = nullptr) override;
243
244 const CPLString& GetURL() const override { return m_osURL; }
245 const CPLString& GetBucket() const { return m_osBucket; }
246 const CPLString& GetObjectKey() const { return m_osObjectKey; }
247 const CPLString& GetEndpoint()const { return m_osEndpoint; }
248 const CPLString& GetRegion() const { return m_osRegion; }
249 const CPLString& GetRequestPayer() const { return m_osRequestPayer; }
250 bool GetVirtualHosting() const { return m_bUseVirtualHosting; }
251 void SetEndpoint(const CPLString &osStr);
252 void SetRegion(const CPLString &osStr);
253 void SetRequestPayer(const CPLString &osStr);
254 void SetVirtualHosting(bool b);
255
256 CPLString GetCopySourceHeader() const override { return "x-amz-copy-source"; }
257 const char* GetMetadataDirectiveREPLACE() const override { return "x-amz-metadata-directive: REPLACE"; }
258
259 CPLString GetSignedURL(CSLConstList papszOptions);
260
261 static void CleanMutex();
262 static void ClearCache();
263 };
264
265 class VSIS3UpdateParams
266 {
267 public:
268 CPLString m_osRegion{};
269 CPLString m_osEndpoint{};
270 CPLString m_osRequestPayer{};
271 bool m_bUseVirtualHosting = false;
272
273 VSIS3UpdateParams() = default;
274
275 explicit VSIS3UpdateParams(const VSIS3HandleHelper* poHelper) :
276 m_osRegion(poHelper->GetRegion()),
277 m_osEndpoint(poHelper->GetEndpoint()),
278 m_osRequestPayer(poHelper->GetRequestPayer()),
279 m_bUseVirtualHosting(poHelper->GetVirtualHosting()) {}
280
281 void UpdateHandlerHelper(VSIS3HandleHelper* poHelper) {
282 poHelper->SetRegion(m_osRegion);
283 poHelper->SetEndpoint(m_osEndpoint);
284 poHelper->SetRequestPayer(m_osRequestPayer);
285 poHelper->SetVirtualHosting(m_bUseVirtualHosting);
286 }
287
288 static std::mutex gsMutex;
289 static std::map< CPLString, VSIS3UpdateParams > goMapBucketsToS3Params;
290 static void UpdateMapFromHandle( IVSIS3LikeHandleHelper* poHandleHelper );
291 static void UpdateHandleFromMap( IVSIS3LikeHandleHelper* poHandleHelper );
292 static void ClearCache();
293 };
294
295 #endif /* HAVE_CURL */
296
297 #endif /* #ifndef DOXYGEN_SKIP */
298
299 #endif /* CPL_AWS_INCLUDED_H */
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:320
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1056
cpl_string.h
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:230
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:930