GDAL

cpl_aws.h
1 /**********************************************************************
2 * $Id$
3 *
4 * Name: cpl_aws.h
5 * Project: CPL - Common Portability Library
6 * Purpose: Amazon Web Services routines
7 * Author: Even Rouault <even.rouault at spatialys.com>
8 *
9 **********************************************************************
10 * Copyright (c) 2015, Even Rouault <even.rouault at spatialys.com>
11 *
12 * Permission is hereby granted, free of charge, to any person obtaining a
13 * copy of this software and associated documentation files (the "Software"),
14 * to deal in the Software without restriction, including without limitation
15 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
16 * and/or sell copies of the Software, and to permit persons to whom the
17 * Software is furnished to do so, subject to the following conditions:
18 *
19 * The above copyright notice and this permission notice shall be included
20 * in all copies or substantial portions of the Software.
21 *
22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
23 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
25 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
28 * DEALINGS IN THE SOFTWARE.
29 ****************************************************************************/
30
31 #ifndef CPL_AWS_INCLUDED_H
32 #define CPL_AWS_INCLUDED_H
33
34 #ifndef DOXYGEN_SKIP
35
36 #ifdef HAVE_CURL
37
38 #include <cstddef>
39 #include <mutex>
40
41 #include "cpl_string.h"
42
43 #include <curl/curl.h>
44 #include <map>
45
46 CPLString CPLGetLowerCaseHexSHA256(const void *pabyData, size_t nBytes);
47 CPLString CPLGetLowerCaseHexSHA256(const CPLString &osStr);
48
49 CPLString CPLGetAWS_SIGN4_Timestamp(GIntBig timestamp);
50
51 CPLString CPLAWSURLEncode(const CPLString &osURL, bool bEncodeSlash = true);
52
53 CPLString CPLAWSGetHeaderVal(const struct curl_slist *psExistingHeaders,
54 const char *pszKey);
55
56 CPLString CPLGetAWS_SIGN4_Signature(
57 const CPLString &osSecretAccessKey, const CPLString &osAccessToken,
58 const CPLString &osRegion, const CPLString &osRequestPayer,
59 const CPLString &osService, const CPLString &osVerb,
60 const struct curl_slist *psExistingHeaders, const CPLString &osHost,
61 const CPLString &osCanonicalURI, const CPLString &osCanonicalQueryString,
62 const CPLString &osXAMZContentSHA256, bool bAddHeaderAMZContentSHA256,
63 const CPLString &osTimestamp, CPLString &osSignedHeaders);
64
65 CPLString CPLGetAWS_SIGN4_Authorization(
66 const CPLString &osSecretAccessKey, const CPLString &osAccessKeyId,
67 const CPLString &osAccessToken, const CPLString &osRegion,
68 const CPLString &osRequestPayer, const CPLString &osService,
69 const CPLString &osVerb, const struct curl_slist *psExistingHeaders,
70 const CPLString &osHost, const CPLString &osCanonicalURI,
71 const CPLString &osCanonicalQueryString,
72 const CPLString &osXAMZContentSHA256, bool bAddHeaderAMZContentSHA256,
73 const CPLString &osTimestamp);
74
75 class IVSIS3LikeHandleHelper
76 {
77 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandleHelper)
78
79 protected:
80 std::map<CPLString, CPLString> m_oMapQueryParameters{};
81
82 virtual void RebuildURL() = 0;
83 CPLString GetQueryString(bool bAddEmptyValueAfterEqual) const;
84
85 public:
86 IVSIS3LikeHandleHelper() = default;
87 virtual ~IVSIS3LikeHandleHelper() = default;
88
89 void ResetQueryParameters();
90 void AddQueryParameter(const CPLString &osKey, const CPLString &osValue);
91
92 virtual struct curl_slist *
93 GetCurlHeaders(const CPLString &osVerb,
94 const struct curl_slist *psExistingHeaders,
95 const void *pabyDataContent = nullptr,
96 size_t nBytesContent = 0) const = 0;
97
98 virtual bool AllowAutomaticRedirection()
99 {
100 return true;
101 }
102 virtual bool CanRestartOnError(const char *, const char * /* pszHeaders*/,
103 bool /*bSetError*/)
104 {
105 return false;
106 }
107
108 virtual const CPLString &GetURL() const = 0;
109 CPLString GetURLNoKVP() const;
110
111 virtual CPLString GetCopySourceHeader() const
112 {
113 return std::string();
114 }
115 virtual const char *GetMetadataDirectiveREPLACE() const
116 {
117 return "";
118 }
119
120 static bool GetBucketAndObjectKey(const char *pszURI,
121 const char *pszFSPrefix,
122 bool bAllowNoObject,
123 CPLString &osBucketOut,
124 CPLString &osObjectKeyOut);
125
126 static CPLString
127 BuildCanonicalizedHeaders(std::map<CPLString, CPLString> &oSortedMapHeaders,
128 const struct curl_slist *psExistingHeaders,
129 const char *pszHeaderPrefix);
130
131 static CPLString GetRFC822DateTime();
132 };
133
134 enum class AWSCredentialsSource
135 {
136 REGULAR, // credentials from env variables or ~/.aws/crediential
137 EC2, // credentials from EC2 private networking
138 WEB_IDENTITY, // credentials from Web Identity Token
139 // See
140 // https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html
141 ASSUMED_ROLE // credentials from an STS assumed role
142 // See
143 // https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-cli.html
144 // and
145 // https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html
146 };
147
148 class VSIS3HandleHelper final : public IVSIS3LikeHandleHelper
149 {
150 CPL_DISALLOW_COPY_ASSIGN(VSIS3HandleHelper)
151
152 CPLString m_osURL{};
153 mutable CPLString m_osSecretAccessKey{};
154 mutable CPLString m_osAccessKeyId{};
155 mutable CPLString m_osSessionToken{};
156 CPLString m_osEndpoint{};
157 CPLString m_osRegion{};
158 CPLString m_osRequestPayer{};
159 CPLString m_osBucket{};
160 CPLString m_osObjectKey{};
161 bool m_bUseHTTPS = false;
162 bool m_bUseVirtualHosting = false;
163 AWSCredentialsSource m_eCredentialsSource = AWSCredentialsSource::REGULAR;
164
165 void RebuildURL() override;
166
167 static bool GetOrRefreshTemporaryCredentialsForRole(
168 bool bForceRefresh, CPLString &osSecretAccessKey,
169 CPLString &osAccessKeyId, CPLString &osSessionToken,
170 CPLString &osRegion);
171
172 static bool GetConfigurationFromAssumeRoleWithWebIdentity(
173 bool bForceRefresh, const std::string &osPathForOption,
174 const std::string &osRoleArnIn,
175 const std::string &osWebIdentityTokenFileIn,
176 CPLString &osSecretAccessKey, CPLString &osAccessKeyId,
177 CPLString &osSessionToken);
178
179 static bool GetConfigurationFromEC2(bool bForceRefresh,
180 const std::string &osPathForOption,
181 CPLString &osSecretAccessKey,
182 CPLString &osAccessKeyId,
183 CPLString &osSessionToken);
184
185 static bool GetConfigurationFromAWSConfigFiles(
186 const std::string &osPathForOption, const char *pszProfile,
187 CPLString &osSecretAccessKey, CPLString &osAccessKeyId,
188 CPLString &osSessionToken, CPLString &osRegion,
189 CPLString &osCredentials, CPLString &osRoleArn,
190 CPLString &osSourceProfile, CPLString &osExternalId,
191 CPLString &osMFASerial, CPLString &osRoleSessionName,
192 CPLString &osWebIdentityTokenFile);
193
194 static bool GetConfiguration(const std::string &osPathForOption,
195 CSLConstList papszOptions,
196 CPLString &osSecretAccessKey,
197 CPLString &osAccessKeyId,
198 CPLString &osSessionToken, CPLString &osRegion,
199 AWSCredentialsSource &eCredentialsSource);
200
201 void RefreshCredentials(const std::string &osPathForOption,
202 bool bForceRefresh) const;
203
204 protected:
205 public:
206 VSIS3HandleHelper(const CPLString &osSecretAccessKey,
207 const CPLString &osAccessKeyId,
208 const CPLString &osSessionToken,
209 const CPLString &osEndpoint, const CPLString &osRegion,
210 const CPLString &osRequestPayer,
211 const CPLString &osBucket, const CPLString &osObjectKey,
212 bool bUseHTTPS, bool bUseVirtualHosting,
213 AWSCredentialsSource eCredentialsSource);
214 ~VSIS3HandleHelper();
215
216 static VSIS3HandleHelper *BuildFromURI(const char *pszURI,
217 const char *pszFSPrefix,
218 bool bAllowNoObject,
219 CSLConstList papszOptions = nullptr);
220 static CPLString BuildURL(const CPLString &osEndpoint,
221 const CPLString &osBucket,
222 const CPLString &osObjectKey, bool bUseHTTPS,
223 bool bUseVirtualHosting);
224
225 struct curl_slist *
226 GetCurlHeaders(const CPLString &osVerb,
227 const struct curl_slist *psExistingHeaders,
228 const void *pabyDataContent = nullptr,
229 size_t nBytesContent = 0) const override;
230
231 bool AllowAutomaticRedirection() override
232 {
233 return false;
234 }
235 bool CanRestartOnError(const char *, const char *pszHeaders,
236 bool bSetError) override;
237
238 const CPLString &GetURL() const override
239 {
240 return m_osURL;
241 }
242 const CPLString &GetBucket() const
243 {
244 return m_osBucket;
245 }
246 const CPLString &GetObjectKey() const
247 {
248 return m_osObjectKey;
249 }
250 const CPLString &GetEndpoint() const
251 {
252 return m_osEndpoint;
253 }
254 const CPLString &GetRegion() const
255 {
256 return m_osRegion;
257 }
258 const CPLString &GetRequestPayer() const
259 {
260 return m_osRequestPayer;
261 }
262 bool GetVirtualHosting() const
263 {
264 return m_bUseVirtualHosting;
265 }
266 void SetEndpoint(const CPLString &osStr);
267 void SetRegion(const CPLString &osStr);
268 void SetRequestPayer(const CPLString &osStr);
269 void SetVirtualHosting(bool b);
270
271 CPLString GetCopySourceHeader() const override
272 {
273 return "x-amz-copy-source";
274 }
275 const char *GetMetadataDirectiveREPLACE() const override
276 {
277 return "x-amz-metadata-directive: REPLACE";
278 }
279
280 CPLString GetSignedURL(CSLConstList papszOptions);
281
282 static void CleanMutex();
283 static void ClearCache();
284 };
285
286 class VSIS3UpdateParams
287 {
288 private:
289 CPLString m_osRegion{};
290 CPLString m_osEndpoint{};
291 CPLString m_osRequestPayer{};
292 bool m_bUseVirtualHosting = false;
293
294 explicit VSIS3UpdateParams(const VSIS3HandleHelper *poHelper)
295 : m_osRegion(poHelper->GetRegion()),
296 m_osEndpoint(poHelper->GetEndpoint()),
297 m_osRequestPayer(poHelper->GetRequestPayer()),
298 m_bUseVirtualHosting(poHelper->GetVirtualHosting())
299 {
300 }
301
302 void UpdateHandlerHelper(VSIS3HandleHelper *poHelper)
303 {
304 poHelper->SetRegion(m_osRegion);
305 poHelper->SetEndpoint(m_osEndpoint);
306 poHelper->SetRequestPayer(m_osRequestPayer);
307 poHelper->SetVirtualHosting(m_bUseVirtualHosting);
308 }
309
310 static std::mutex gsMutex;
311 static std::map<CPLString, VSIS3UpdateParams> goMapBucketsToS3Params;
312
313 public:
314 VSIS3UpdateParams() = default;
315
316 static void UpdateMapFromHandle(VSIS3HandleHelper *poS3HandleHelper);
317 static void UpdateHandleFromMap(VSIS3HandleHelper *poS3HandleHelper);
318 static void ClearCache();
319 };
320
321 #endif /* HAVE_CURL */
322
323 #endif /* #ifndef DOXYGEN_SKIP */
324
325 #endif /* CPL_AWS_INCLUDED_H */
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:311
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1178
cpl_string.h
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:226
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1042