Skip to content

Commit

Permalink
/vsigs/: make GetFileMetadata('/vsigs/bucket', NULL, NULL) work if us…
Browse files Browse the repository at this point in the history 
…ing OAuth2 auth
  • Loading branch information
@rouault
rouault committed Oct 25, 2024
1 parent cbdc8a4 commit 62e2c6e
Show file tree
Hide file tree
Showing 3 changed files with 238 additions and 7 deletions.
101 changes: 101 additions & 0 deletions autotest/gcore/vsigs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -692,6 +692,107 @@ def test_vsigs_headers(gs_test_config, webserver_port):
)


###############################################################################
# Test GetFileMetadata() on root of bucket with OAuth2


@gdaltest.enable_exceptions()
def test_vsigs_GetFileMetadatabucket_root_oauth2(
gs_test_config, webserver_port, tmp_vsimem
):

gdal.VSICurlClearCache()

service_account_filename = str(tmp_vsimem / "service_account.json")
gdal.FileFromMemBuffer(
service_account_filename,
"""{
"private_key": "-----BEGIN PRIVATE KEY-----\nMIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAOlwJQLLDG1HeLrk\nVNcFR5Qptto/rJE5emRuy0YmkVINT4uHb1be7OOo44C2Ev8QPVtNHHS2XwCY5gTm\ni2RfIBLv+VDMoVQPqqE0LHb0WeqGmM5V1tHbmVnIkCcKMn3HpK30grccuBc472LQ\nDVkkGqIiGu0qLAQ89JP/r0LWWySRAgMBAAECgYAWjsS00WRBByAOh1P/dz4kfidy\nTabiXbiLDf3MqJtwX2Lpa8wBjAc+NKrPXEjXpv0W3ou6Z4kkqKHJpXGg4GRb4N5I\n2FA+7T1lA0FCXa7dT2jvgJLgpBepJu5b//tqFqORb4A4gMZw0CiPN3sUsWsSw5Hd\nDrRXwp6sarzG77kvZQJBAPgysAmmXIIp9j1hrFSkctk4GPkOzZ3bxKt2Nl4GFrb+\nbpKSon6OIhP1edrxTz1SMD1k5FiAAVUrMDKSarbh5osCQQDwxq4Tvf/HiYz79JBg\nWz5D51ySkbg01dOVgFW3eaYAdB6ta/o4vpHhnbrfl6VO9oUb3QR4hcrruwnDHsw3\n4mDTAkEA9FPZjbZSTOSH/cbgAXbdhE4/7zWOXj7Q7UVyob52r+/p46osAk9i5qj5\nKvnv2lrFGDrwutpP9YqNaMtP9/aLnwJBALLWf9n+GAv3qRZD0zEe1KLPKD1dqvrj\nj+LNjd1Xp+tSVK7vMs4PDoAMDg+hrZF3HetSQM3cYpqxNFEPgRRJOy0CQQDQlZHI\nyzpSgEiyx8O3EK1iTidvnLXbtWabvjZFfIE/0OhfBmN225MtKG3YLV2HoUvpajLq\ngwE6fxOLyJDxuWRf\n-----END PRIVATE KEY-----\n",
"client_email": "CLIENT_EMAIL",
"type": "service_account"
}""",
)

gdal.SetPathSpecificOption(
"/vsigs/gs_fake_bucket",
"GOOGLE_APPLICATION_CREDENTIALS",
service_account_filename,
)

try:
with gdaltest.config_options(
{
"GO2A_AUD": "http://localhost:%d/oauth2/v4/token" % webserver_port,
"GOA2_NOW": "123456",
},
thread_local=False,
):

gdal.VSICurlClearCache()

handler = webserver.SequentialHandler()

def method(request):
request.send_response(200)
request.send_header("Content-type", "text/plain")
content = """{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 3600,
}"""
request.send_header("Content-Length", len(content))
request.end_headers()
request.wfile.write(content.encode("ascii"))

handler.add("POST", "/oauth2/v4/token", custom_method=method)

handler.add(
"GET",
"/storage/v1/b/gs_fake_bucket",
200,
{"Content-type": "application/json"},
'{"foo":"bar"}',
)
try:
with webserver.install_http_handler(handler):
md = gdal.GetFileMetadata("/vsigs/gs_fake_bucket/", None)

except Exception:
if (
gdal.GetLastErrorMsg().find("CPLRSASHA256Sign() not implemented")
>= 0
):
pytest.skip("CPLRSASHA256Sign() not implemented")

assert md == {"foo": "bar"}
finally:
gdal.SetPathSpecificOption(
"/vsigs/gs_fake_bucket", "GOOGLE_APPLICATION_CREDENTIALS", None
)


###############################################################################
# Test GetFileMetadata() on root of bucket with non-OAuth2 (does not work)


def test_vsigs_GetFileMetadatabucket_root_not_oauth2(gs_test_config, webserver_port):

gdal.VSICurlClearCache()

with gdaltest.config_options(
{
"GS_SECRET_ACCESS_KEY": "GS_SECRET_ACCESS_KEY",
"GS_ACCESS_KEY_ID": "GS_ACCESS_KEY_ID",
},
thread_local=False,
):

handler = webserver.SequentialHandler()
with webserver.install_http_handler(handler):
md = gdal.GetFileMetadata("/vsigs/gs_fake_bucket/", None)
assert md == {}


###############################################################################
# Read credentials with OAuth2 refresh_token

Expand Down
23 changes: 17 additions & 6 deletions port/cpl_vsil.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1231,24 +1231,35 @@ int VSIStatExL(const char *pszFilename, VSIStatBufL *psStatBuf, int nFlags)
* Implemented currently only for network-like filesystems, or starting
* with GDAL 3.7 for /vsizip/
*
* Starting with GDAL 3.11, calling it with pszFilename being the root of a
* /vsigs/ bucket and pszDomain == nullptr, and when authenticated through
* OAuth2, will result in returning the result of a "Buckets: get"
* operation (https://cloud.google.com/storage/docs/json_api/v1/buckets/get),
* with the keys of the top-level JSON document as keys of the key=value pairs
* returned by this function.
*
* @param pszFilename the path of the filesystem object to be queried.
* UTF-8 encoded.
* @param pszDomain Metadata domain to query. Depends on the file system.
* The following are supported:
* The following ones are supported:
* <ul>
* <li>HEADERS: to get HTTP headers for network-like filesystems (/vsicurl/,
* /vsis3/, /vsgis/, etc)</li> <li>TAGS: <ul> <li>/vsis3/: to get S3 Object
* tagging information</li> <li>/vsiaz/: to get blob tags. Refer to
* https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-tags</li>
* </ul>
* /vsis3/, /vsgis/, etc)</li>
* <li>TAGS:
* <ul>
* <li>/vsis3/: to get S3 Object tagging information</li>
* <li>/vsiaz/: to get blob tags. Refer to
* https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-tags
* </li>
* </ul>
* </li>
* <li>STATUS: specific to /vsiadls/: returns all system defined properties for
* a path (seems in practice to be a subset of HEADERS)</li> <li>ACL: specific
* to /vsiadls/ and /vsigs/: returns the access control list for a path. For
* /vsigs/, a single XML=xml_content string is returned. Refer to
* https://cloud.google.com/storage/docs/xml-api/get-object-acls
* </li>
* <li>METADATA: specific to /vsiaz/: to set blob metadata. Refer to
* <li>METADATA: specific to /vsiaz/: to get blob metadata. Refer to
* https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-metadata.
* Note: this will be a subset of what pszDomain=HEADERS returns</li>
* <li>ZIP: specific to /vsizip/: to obtain ZIP specific metadata, in particular
Expand Down
121 changes: 120 additions & 1 deletion port/cpl_vsil_gs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include "cpl_port.h"
#include "cpl_http.h"
#include "cpl_minixml.h"
#include "cpl_json.h"
#include "cpl_vsil_curl_priv.h"
#include "cpl_vsil_curl_class.h"

Expand Down Expand Up @@ -331,6 +332,124 @@ char **VSIGSFSHandler::GetFileMetadata(const char *pszFilename,
if (!STARTS_WITH_CI(pszFilename, GetFSPrefix().c_str()))
return nullptr;

if (pszDomain == nullptr)
{
// Handle case of requesting GetFileMetadata() on the bucket root
std::string osFilename(pszFilename);
if (osFilename.back() == '/')
osFilename.pop_back();
if (osFilename.find('/', GetFSPrefix().size()) == std::string::npos)
{
const std::string osBucket =
osFilename.substr(GetFSPrefix().size());
const std::string osResource =
std::string("storage/v1/b/").append(osBucket);

auto poHandleHelper = std::unique_ptr<VSIGSHandleHelper>(
VSIGSHandleHelper::BuildFromURI(osResource.c_str(),
GetFSPrefix().c_str(),
osBucket.c_str()));
if (!poHandleHelper)
return nullptr;

// The JSON API cannot be used with HMAC keys
if (poHandleHelper->UsesHMACKey())
{
CPLDebug(GetDebugKey(),
"GetFileMetadata() on bucket "
"only available for OAuth2 authentication");
return VSICurlFilesystemHandlerBase::GetFileMetadata(
pszFilename, pszDomain, papszOptions);
}

NetworkStatisticsFileSystem oContextFS(GetFSPrefix().c_str());
NetworkStatisticsAction oContextAction("GetFileMetadata");

const CPLStringList aosHTTPOptions(
CPLHTTPGetOptionsFromEnv(pszFilename));
const CPLHTTPRetryParameters oRetryParameters(aosHTTPOptions);
CPLHTTPRetryContext oRetryContext(oRetryParameters);

bool bRetry;
CPLStringList aosResult;
do
{
bRetry = false;
CURL *hCurlHandle = curl_easy_init();

struct curl_slist *headers =
static_cast<struct curl_slist *>(CPLHTTPSetOptions(
hCurlHandle, poHandleHelper->GetURL().c_str(),
aosHTTPOptions.List()));
headers = VSICurlMergeHeaders(
headers, poHandleHelper->GetCurlHeaders("GET", headers));

CurlRequestHelper requestHelper;
const long response_code = requestHelper.perform(
hCurlHandle, headers, this, poHandleHelper.get());

NetworkStatisticsLogger::LogGET(
requestHelper.sWriteFuncData.nSize);

if (response_code != 200 ||
requestHelper.sWriteFuncData.pBuffer == nullptr)
{
// Look if we should attempt a retry
if (oRetryContext.CanRetry(
static_cast<int>(response_code),
requestHelper.sWriteFuncHeaderData.pBuffer,
requestHelper.szCurlErrBuf))
{
CPLError(CE_Warning, CPLE_AppDefined,
"HTTP error code: %d - %s. "
"Retrying again in %.1f secs",
static_cast<int>(response_code),
poHandleHelper->GetURL().c_str(),
oRetryContext.GetCurrentDelay());
CPLSleep(oRetryContext.GetCurrentDelay());
bRetry = true;
}
else
{
CPLDebug(GetDebugKey(), "%s",
requestHelper.sWriteFuncData.pBuffer
? requestHelper.sWriteFuncData.pBuffer
: "(null)");
CPLError(CE_Failure, CPLE_AppDefined,
"GetFileMetadata failed");
}
}
else
{
CPLJSONDocument oDoc;
if (oDoc.LoadMemory(
reinterpret_cast<const GByte *>(
requestHelper.sWriteFuncData.pBuffer),
static_cast<int>(
requestHelper.sWriteFuncData.nSize)) &&
oDoc.GetRoot().GetType() == CPLJSONObject::Type::Object)
{
for (const auto &oObj : oDoc.GetRoot().GetChildren())
{
aosResult.SetNameValue(oObj.GetName().c_str(),
oObj.ToString().c_str());
}
}
else
{
// Shouldn't happen normally
aosResult.SetNameValue(
"DATA", requestHelper.sWriteFuncData.pBuffer);
}
}

curl_easy_cleanup(hCurlHandle);
} while (bRetry);

return aosResult.StealList();
}
}

if (pszDomain == nullptr || !EQUAL(pszDomain, "ACL"))
{
return VSICurlFilesystemHandlerBase::GetFileMetadata(
Expand Down Expand Up @@ -404,7 +523,7 @@ char **VSIGSFSHandler::GetFileMetadata(const char *pszFilename,

curl_easy_cleanup(hCurlHandle);
} while (bRetry);
return CSLDuplicate(aosResult.List());
return aosResult.StealList();
}

/************************************************************************/
Expand Down

0 comments on commit 62e2c6e

Please sign in to comment.