GDAL

cpl_aws.h
1 /**********************************************************************
2 * $Id$
3 *
4 * Name: cpl_aws.h
5 * Project: CPL - Common Portability Library
6 * Purpose: Amazon Web Services routines
7 * Author: Even Rouault <even.rouault at spatialys.com>
8 *
9 **********************************************************************
10 * Copyright (c) 2015, Even Rouault <even.rouault at spatialys.com>
11 *
12 * Permission is hereby granted, free of charge, to any person obtaining a
13 * copy of this software and associated documentation files (the "Software"),
14 * to deal in the Software without restriction, including without limitation
15 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
16 * and/or sell copies of the Software, and to permit persons to whom the
17 * Software is furnished to do so, subject to the following conditions:
18 *
19 * The above copyright notice and this permission notice shall be included
20 * in all copies or substantial portions of the Software.
21 *
22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
23 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
25 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
28 * DEALINGS IN THE SOFTWARE.
29 ****************************************************************************/
30
31 #ifndef CPL_AWS_INCLUDED_H
32 #define CPL_AWS_INCLUDED_H
33
34 #ifndef DOXYGEN_SKIP
35
36 #ifdef HAVE_CURL
37
38 #include <cstddef>
39 #include <mutex>
40
41 #include "cpl_string.h"
42
43 #include <curl/curl.h>
44 #include <map>
45
46 CPLString CPLGetLowerCaseHexSHA256(const void *pabyData, size_t nBytes);
47 CPLString CPLGetLowerCaseHexSHA256(const CPLString &osStr);
48
49 CPLString CPLGetAWS_SIGN4_Timestamp(GIntBig timestamp);
50
51 CPLString CPLAWSURLEncode(const CPLString &osURL, bool bEncodeSlash = true);
52
53 CPLString CPLAWSGetHeaderVal(const struct curl_slist *psExistingHeaders,
54 const char *pszKey);
55
56 CPLString CPLGetAWS_SIGN4_Signature(
57 const CPLString &osSecretAccessKey, const CPLString &osAccessToken,
58 const CPLString &osRegion, const CPLString &osRequestPayer,
59 const CPLString &osService, const CPLString &osVerb,
60 const struct curl_slist *psExistingHeaders, const CPLString &osHost,
61 const CPLString &osCanonicalURI, const CPLString &osCanonicalQueryString,
62 const CPLString &osXAMZContentSHA256, bool bAddHeaderAMZContentSHA256,
63 const CPLString &osTimestamp, CPLString &osSignedHeaders);
64
65 CPLString CPLGetAWS_SIGN4_Authorization(
66 const CPLString &osSecretAccessKey, const CPLString &osAccessKeyId,
67 const CPLString &osAccessToken, const CPLString &osRegion,
68 const CPLString &osRequestPayer, const CPLString &osService,
69 const CPLString &osVerb, const struct curl_slist *psExistingHeaders,
70 const CPLString &osHost, const CPLString &osCanonicalURI,
71 const CPLString &osCanonicalQueryString,
72 const CPLString &osXAMZContentSHA256, bool bAddHeaderAMZContentSHA256,
73 const CPLString &osTimestamp);
74
75 class IVSIS3LikeHandleHelper
76 {
77 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandleHelper)
78
79 protected:
80 std::map<CPLString, CPLString> m_oMapQueryParameters{};
81
82 virtual void RebuildURL() = 0;
83 CPLString GetQueryString(bool bAddEmptyValueAfterEqual) const;
84
85 public:
86 IVSIS3LikeHandleHelper() = default;
87 virtual ~IVSIS3LikeHandleHelper() = default;
88
89 void ResetQueryParameters();
90 void AddQueryParameter(const CPLString &osKey, const CPLString &osValue);
91
92 virtual struct curl_slist *
93 GetCurlHeaders(const CPLString &osVerb,
94 const struct curl_slist *psExistingHeaders,
95 const void *pabyDataContent = nullptr,
96 size_t nBytesContent = 0) const = 0;
97
98 virtual bool AllowAutomaticRedirection()
99 {
100 return true;
101 }
102 virtual bool CanRestartOnError(const char *, const char * /* pszHeaders*/,
103 bool /*bSetError*/,
104 bool * /*pbUpdateMap*/ = nullptr)
105 {
106 return false;
107 }
108
109 virtual const CPLString &GetURL() const = 0;
110 CPLString GetURLNoKVP() const;
111
112 virtual CPLString GetCopySourceHeader() const
113 {
114 return std::string();
115 }
116 virtual const char *GetMetadataDirectiveREPLACE() const
117 {
118 return "";
119 }
120
121 static bool GetBucketAndObjectKey(const char *pszURI,
122 const char *pszFSPrefix,
123 bool bAllowNoObject,
124 CPLString &osBucketOut,
125 CPLString &osObjectKeyOut);
126
127 static CPLString
128 BuildCanonicalizedHeaders(std::map<CPLString, CPLString> &oSortedMapHeaders,
129 const struct curl_slist *psExistingHeaders,
130 const char *pszHeaderPrefix);
131
132 static CPLString GetRFC822DateTime();
133 };
134
135 enum class AWSCredentialsSource
136 {
137 REGULAR, // credentials from env variables or ~/.aws/crediential
138 EC2, // credentials from EC2 private networking
139 WEB_IDENTITY, // credentials from Web Identity Token
140 // See
141 // https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html
142 ASSUMED_ROLE // credentials from an STS assumed role
143 // See
144 // https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-cli.html
145 // and
146 // https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html
147 };
148
149 class VSIS3HandleHelper final : public IVSIS3LikeHandleHelper
150 {
151 CPL_DISALLOW_COPY_ASSIGN(VSIS3HandleHelper)
152
153 CPLString m_osURL{};
154 mutable CPLString m_osSecretAccessKey{};
155 mutable CPLString m_osAccessKeyId{};
156 mutable CPLString m_osSessionToken{};
157 CPLString m_osEndpoint{};
158 CPLString m_osRegion{};
159 CPLString m_osRequestPayer{};
160 CPLString m_osBucket{};
161 CPLString m_osObjectKey{};
162 bool m_bUseHTTPS = false;
163 bool m_bUseVirtualHosting = false;
164 AWSCredentialsSource m_eCredentialsSource = AWSCredentialsSource::REGULAR;
165
166 void RebuildURL() override;
167
168 static bool GetOrRefreshTemporaryCredentialsForRole(
169 bool bForceRefresh, CPLString &osSecretAccessKey,
170 CPLString &osAccessKeyId, CPLString &osSessionToken,
171 CPLString &osRegion);
172
173 static bool GetConfigurationFromAssumeRoleWithWebIdentity(
174 bool bForceRefresh, const std::string &osPathForOption,
175 const std::string &osRoleArnIn,
176 const std::string &osWebIdentityTokenFileIn,
177 CPLString &osSecretAccessKey, CPLString &osAccessKeyId,
178 CPLString &osSessionToken);
179
180 static bool GetConfigurationFromEC2(bool bForceRefresh,
181 const std::string &osPathForOption,
182 CPLString &osSecretAccessKey,
183 CPLString &osAccessKeyId,
184 CPLString &osSessionToken);
185
186 static bool GetConfigurationFromAWSConfigFiles(
187 const std::string &osPathForOption, const char *pszProfile,
188 CPLString &osSecretAccessKey, CPLString &osAccessKeyId,
189 CPLString &osSessionToken, CPLString &osRegion,
190 CPLString &osCredentials, CPLString &osRoleArn,
191 CPLString &osSourceProfile, CPLString &osExternalId,
192 CPLString &osMFASerial, CPLString &osRoleSessionName,
193 CPLString &osWebIdentityTokenFile);
194
195 static bool GetConfiguration(const std::string &osPathForOption,
196 CSLConstList papszOptions,
197 CPLString &osSecretAccessKey,
198 CPLString &osAccessKeyId,
199 CPLString &osSessionToken, CPLString &osRegion,
200 AWSCredentialsSource &eCredentialsSource);
201
202 void RefreshCredentials(const std::string &osPathForOption,
203 bool bForceRefresh) const;
204
205 protected:
206 public:
207 VSIS3HandleHelper(const CPLString &osSecretAccessKey,
208 const CPLString &osAccessKeyId,
209 const CPLString &osSessionToken,
210 const CPLString &osEndpoint, const CPLString &osRegion,
211 const CPLString &osRequestPayer,
212 const CPLString &osBucket, const CPLString &osObjectKey,
213 bool bUseHTTPS, bool bUseVirtualHosting,
214 AWSCredentialsSource eCredentialsSource);
215 ~VSIS3HandleHelper();
216
217 static VSIS3HandleHelper *BuildFromURI(const char *pszURI,
218 const char *pszFSPrefix,
219 bool bAllowNoObject,
220 CSLConstList papszOptions = nullptr);
221 static CPLString BuildURL(const CPLString &osEndpoint,
222 const CPLString &osBucket,
223 const CPLString &osObjectKey, bool bUseHTTPS,
224 bool bUseVirtualHosting);
225
226 struct curl_slist *
227 GetCurlHeaders(const CPLString &osVerb,
228 const struct curl_slist *psExistingHeaders,
229 const void *pabyDataContent = nullptr,
230 size_t nBytesContent = 0) const override;
231
232 bool AllowAutomaticRedirection() override
233 {
234 return false;
235 }
236 bool CanRestartOnError(const char *, const char *pszHeaders, bool bSetError,
237 bool *pbUpdateMap = nullptr) override;
238
239 const CPLString &GetURL() const override
240 {
241 return m_osURL;
242 }
243 const CPLString &GetBucket() const
244 {
245 return m_osBucket;
246 }
247 const CPLString &GetObjectKey() const
248 {
249 return m_osObjectKey;
250 }
251 const CPLString &GetEndpoint() const
252 {
253 return m_osEndpoint;
254 }
255 const CPLString &GetRegion() const
256 {
257 return m_osRegion;
258 }
259 const CPLString &GetRequestPayer() const
260 {
261 return m_osRequestPayer;
262 }
263 bool GetVirtualHosting() const
264 {
265 return m_bUseVirtualHosting;
266 }
267 void SetEndpoint(const CPLString &osStr);
268 void SetRegion(const CPLString &osStr);
269 void SetRequestPayer(const CPLString &osStr);
270 void SetVirtualHosting(bool b);
271
272 CPLString GetCopySourceHeader() const override
273 {
274 return "x-amz-copy-source";
275 }
276 const char *GetMetadataDirectiveREPLACE() const override
277 {
278 return "x-amz-metadata-directive: REPLACE";
279 }
280
281 CPLString GetSignedURL(CSLConstList papszOptions);
282
283 static void CleanMutex();
284 static void ClearCache();
285 };
286
287 class VSIS3UpdateParams
288 {
289 public:
290 CPLString m_osRegion{};
291 CPLString m_osEndpoint{};
292 CPLString m_osRequestPayer{};
293 bool m_bUseVirtualHosting = false;
294
295 VSIS3UpdateParams() = default;
296
297 explicit VSIS3UpdateParams(const VSIS3HandleHelper *poHelper)
298 : m_osRegion(poHelper->GetRegion()),
299 m_osEndpoint(poHelper->GetEndpoint()),
300 m_osRequestPayer(poHelper->GetRequestPayer()),
301 m_bUseVirtualHosting(poHelper->GetVirtualHosting())
302 {
303 }
304
305 void UpdateHandlerHelper(VSIS3HandleHelper *poHelper)
306 {
307 poHelper->SetRegion(m_osRegion);
308 poHelper->SetEndpoint(m_osEndpoint);
309 poHelper->SetRequestPayer(m_osRequestPayer);
310 poHelper->SetVirtualHosting(m_bUseVirtualHosting);
311 }
312
313 static std::mutex gsMutex;
314 static std::map<CPLString, VSIS3UpdateParams> goMapBucketsToS3Params;
315 static void UpdateMapFromHandle(IVSIS3LikeHandleHelper *poHandleHelper);
316 static void UpdateHandleFromMap(IVSIS3LikeHandleHelper *poHandleHelper);
317 static void ClearCache();
318 };
319
320 #endif /* HAVE_CURL */
321
322 #endif /* #ifndef DOXYGEN_SKIP */
323
324 #endif /* CPL_AWS_INCLUDED_H */
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:311
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1195
cpl_string.h
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:235
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1051