GDAL

cpl_aws.h
1/**********************************************************************
2 * $Id$
3 *
4 * Name: cpl_aws.h
5 * Project: CPL - Common Portability Library
6 * Purpose: Amazon Web Services routines
7 * Author: Even Rouault <even.rouault at spatialys.com>
8 *
9 **********************************************************************
10 * Copyright (c) 2015, Even Rouault <even.rouault at spatialys.com>
11 *
12 * SPDX-License-Identifier: MIT
13 ****************************************************************************/
14
15#ifndef CPL_AWS_INCLUDED_H
16#define CPL_AWS_INCLUDED_H
17
18#ifndef DOXYGEN_SKIP
19
20#ifdef HAVE_CURL
21
22#include <cstddef>
23#include <mutex>
24
25#include "cpl_string.h"
26
27#include <curl/curl.h>
28#include <map>
29
30std::string CPLGetLowerCaseHexSHA256(const void *pabyData, size_t nBytes);
31std::string CPLGetLowerCaseHexSHA256(const std::string &osStr);
32
33std::string CPLGetAWS_SIGN4_Timestamp(GIntBig timestamp);
34
35std::string CPLAWSURLEncode(const std::string &osURL, bool bEncodeSlash = true);
36
37std::string CPLAWSGetHeaderVal(const struct curl_slist *psExistingHeaders,
38 const char *pszKey);
39
40std::string CPLGetAWS_SIGN4_Signature(
41 const std::string &osSecretAccessKey, const std::string &osAccessToken,
42 const std::string &osRegion, const std::string &osRequestPayer,
43 const std::string &osService, const std::string &osVerb,
44 const struct curl_slist *psExistingHeaders, const std::string &osHost,
45 const std::string &osCanonicalURI,
46 const std::string &osCanonicalQueryString,
47 const std::string &osXAMZContentSHA256, bool bAddHeaderAMZContentSHA256,
48 const std::string &osTimestamp, std::string &osSignedHeaders);
49
50std::string CPLGetAWS_SIGN4_Authorization(
51 const std::string &osSecretAccessKey, const std::string &osAccessKeyId,
52 const std::string &osAccessToken, const std::string &osRegion,
53 const std::string &osRequestPayer, const std::string &osService,
54 const std::string &osVerb, const struct curl_slist *psExistingHeaders,
55 const std::string &osHost, const std::string &osCanonicalURI,
56 const std::string &osCanonicalQueryString,
57 const std::string &osXAMZContentSHA256, bool bAddHeaderAMZContentSHA256,
58 const std::string &osTimestamp);
59
60 class IVSIS3LikeHandleHelper
61{
62 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandleHelper)
63
64 protected:
65 std::map<std::string, std::string> m_oMapQueryParameters{};
66
67 virtual void RebuildURL() = 0;
68 std::string GetQueryString(bool bAddEmptyValueAfterEqual) const;
69
70 public:
71 IVSIS3LikeHandleHelper() = default;
72 virtual ~IVSIS3LikeHandleHelper() = default;
73
74 void ResetQueryParameters();
75 void AddQueryParameter(const std::string &osKey,
76 const std::string &osValue);
77
78 virtual struct curl_slist *
79 GetCurlHeaders(const std::string &osVerb,
80 const struct curl_slist *psExistingHeaders,
81 const void *pabyDataContent = nullptr,
82 size_t nBytesContent = 0) const = 0;
83
84 virtual bool AllowAutomaticRedirection()
85 {
86 return true;
87 }
88
89 virtual bool CanRestartOnError(const char *, const char * /* pszHeaders*/,
90 bool /*bSetError*/)
91 {
92 return false;
93 }
94
95 virtual const std::string &GetURL() const = 0;
96 std::string GetURLNoKVP() const;
97
98 virtual std::string GetCopySourceHeader() const
99 {
100 return std::string();
101 }
102
103 virtual const char *GetMetadataDirectiveREPLACE() const
104 {
105 return "";
106 }
107
108 static bool GetBucketAndObjectKey(const char *pszURI,
109 const char *pszFSPrefix,
110 bool bAllowNoObject,
111 std::string &osBucketOut,
112 std::string &osObjectKeyOut);
113
114 static std::string BuildCanonicalizedHeaders(
115 std::map<std::string, std::string> &oSortedMapHeaders,
116 const struct curl_slist *psExistingHeaders,
117 const char *pszHeaderPrefix);
118
119 static std::string GetRFC822DateTime();
120};
121
122 enum class AWSCredentialsSource
123{
124 REGULAR, // credentials from env variables or ~/.aws/crediential
125 EC2, // credentials from EC2 private networking
126 WEB_IDENTITY, // credentials from Web Identity Token
127 // See
128 // https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html
129 ASSUMED_ROLE // credentials from an STS assumed role
130 // See
131 // https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-cli.html
132 // and
133 // https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html
134};
135
136 class VSIS3HandleHelper final : public IVSIS3LikeHandleHelper
137{
138 CPL_DISALLOW_COPY_ASSIGN(VSIS3HandleHelper)
139
140 std::string m_osURL{};
141 mutable std::string m_osSecretAccessKey{};
142 mutable std::string m_osAccessKeyId{};
143 mutable std::string m_osSessionToken{};
144 std::string m_osEndpoint{};
145 std::string m_osRegion{};
146 std::string m_osRequestPayer{};
147 std::string m_osBucket{};
148 std::string m_osObjectKey{};
149 bool m_bUseHTTPS = false;
150 bool m_bUseVirtualHosting = false;
151 AWSCredentialsSource m_eCredentialsSource = AWSCredentialsSource::REGULAR;
152
153 void RebuildURL() override;
154
155 static bool GetOrRefreshTemporaryCredentialsForRole(
156 bool bForceRefresh, std::string &osSecretAccessKey,
157 std::string &osAccessKeyId, std::string &osSessionToken,
158 std::string &osRegion);
159
160 static bool GetConfigurationFromAssumeRoleWithWebIdentity(
161 bool bForceRefresh, const std::string &osPathForOption,
162 const std::string &osRoleArnIn,
163 const std::string &osWebIdentityTokenFileIn,
164 std::string &osSecretAccessKey, std::string &osAccessKeyId,
165 std::string &osSessionToken);
166
167 static bool GetConfigurationFromEC2(bool bForceRefresh,
168 const std::string &osPathForOption,
169 std::string &osSecretAccessKey,
170 std::string &osAccessKeyId,
171 std::string &osSessionToken);
172
173 static bool GetConfigurationFromAWSConfigFiles(
174 const std::string &osPathForOption, const char *pszProfile,
175 std::string &osSecretAccessKey, std::string &osAccessKeyId,
176 std::string &osSessionToken, std::string &osRegion,
177 std::string &osCredentials, std::string &osRoleArn,
178 std::string &osSourceProfile, std::string &osExternalId,
179 std::string &osMFASerial, std::string &osRoleSessionName,
180 std::string &osWebIdentityTokenFile);
181
182 static bool GetConfiguration(const std::string &osPathForOption,
183 CSLConstList papszOptions,
184 std::string &osSecretAccessKey,
185 std::string &osAccessKeyId,
186 std::string &osSessionToken,
187 std::string &osRegion,
188 AWSCredentialsSource &eCredentialsSource);
189
190 void RefreshCredentials(const std::string &osPathForOption,
191 bool bForceRefresh) const;
192
193 protected:
194 public:
195 VSIS3HandleHelper(
196 const std::string &osSecretAccessKey, const std::string &osAccessKeyId,
197 const std::string &osSessionToken, const std::string &osEndpoint,
198 const std::string &osRegion, const std::string &osRequestPayer,
199 const std::string &osBucket, const std::string &osObjectKey,
200 bool bUseHTTPS, bool bUseVirtualHosting,
201 AWSCredentialsSource eCredentialsSource);
202 ~VSIS3HandleHelper();
203
204 static VSIS3HandleHelper *BuildFromURI(const char *pszURI,
205 const char *pszFSPrefix,
206 bool bAllowNoObject,
207 CSLConstList papszOptions = nullptr);
208 static std::string BuildURL(const std::string &osEndpoint,
209 const std::string &osBucket,
210 const std::string &osObjectKey, bool bUseHTTPS,
211 bool bUseVirtualHosting);
212
213 struct curl_slist *
214 GetCurlHeaders(const std::string &osVerb,
215 const struct curl_slist *psExistingHeaders,
216 const void *pabyDataContent = nullptr,
217 size_t nBytesContent = 0) const override;
218
219 bool AllowAutomaticRedirection() override
220 {
221 return false;
222 }
223
224 bool CanRestartOnError(const char *, const char *pszHeaders,
225 bool bSetError) override;
226
227 const std::string &GetURL() const override
228 {
229 return m_osURL;
230 }
231
232 const std::string &GetBucket() const
233 {
234 return m_osBucket;
235 }
236
237 const std::string &GetObjectKey() const
238 {
239 return m_osObjectKey;
240 }
241
242 const std::string &GetEndpoint() const
243 {
244 return m_osEndpoint;
245 }
246
247 const std::string &GetRegion() const
248 {
249 return m_osRegion;
250 }
251
252 const std::string &GetRequestPayer() const
253 {
254 return m_osRequestPayer;
255 }
256
257 bool GetVirtualHosting() const
258 {
259 return m_bUseVirtualHosting;
260 }
261
262 void SetEndpoint(const std::string &osStr);
263 void SetRegion(const std::string &osStr);
264 void SetRequestPayer(const std::string &osStr);
265 void SetVirtualHosting(bool b);
266
267 std::string GetCopySourceHeader() const override
268 {
269 return "x-amz-copy-source";
270 }
271
272 const char *GetMetadataDirectiveREPLACE() const override
273 {
274 return "x-amz-metadata-directive: REPLACE";
275 }
276
277 std::string GetSignedURL(CSLConstList papszOptions);
278
279 static void CleanMutex();
280 static void ClearCache();
281};
282
283 class VSIS3UpdateParams
284{
285 private:
286 std::string m_osRegion{};
287 std::string m_osEndpoint{};
288 std::string m_osRequestPayer{};
289 bool m_bUseVirtualHosting = false;
290
291 explicit VSIS3UpdateParams(const VSIS3HandleHelper *poHelper)
292 : m_osRegion(poHelper->GetRegion()),
293 m_osEndpoint(poHelper->GetEndpoint()),
294 m_osRequestPayer(poHelper->GetRequestPayer()),
295 m_bUseVirtualHosting(poHelper->GetVirtualHosting())
296 {
297 }
298
299 void UpdateHandlerHelper(VSIS3HandleHelper *poHelper)
300 {
301 poHelper->SetRegion(m_osRegion);
302 poHelper->SetEndpoint(m_osEndpoint);
303 poHelper->SetRequestPayer(m_osRequestPayer);
304 poHelper->SetVirtualHosting(m_bUseVirtualHosting);
305 }
306
307 static std::mutex gsMutex;
308 static std::map<std::string, VSIS3UpdateParams> goMapBucketsToS3Params;
309
310 public:
311 VSIS3UpdateParams() = default;
312
313 static void UpdateMapFromHandle(VSIS3HandleHelper *poS3HandleHelper);
314 static void UpdateHandleFromMap(VSIS3HandleHelper *poS3HandleHelper);
315 static void ClearCache();
316};
317
318#endif /* HAVE_CURL */
319
320#endif /* #ifndef DOXYGEN_SKIP */
321
322#endif /* CPL_AWS_INCLUDED_H */
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1030
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1179
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:199
cpl_string.h
Various convenience functions for working with strings and string lists.