diff --git a/credentials/credential_provider.cc b/credentials/credential_provider.cc new file mode 100644 index 00000000..6f1eb5ac --- /dev/null +++ b/credentials/credential_provider.cc @@ -0,0 +1,117 @@ +/* credential_provider.cc + Jeremy Barnes, 5 November 2014 + Copyright (c) 2014 Datacratic Inc. All rights reserved. + + Basic functionality to get credentials. +*/ + +#include "credential_provider.h" +#include + +using namespace std; + +namespace Datacratic { + +/*****************************************************************************/ +/* CREDENTIAL PROVIDER */ +/*****************************************************************************/ + +CredentialProvider:: +~CredentialProvider() +{ +} + +namespace { + +std::mutex providersLock; +std::multimap > providers; + +} // file scope + +void +CredentialProvider:: +registerProvider(const std::string & name, + std::shared_ptr provider) +{ + std::unique_lock guard(providersLock); + + auto prefixes = provider->getResourceTypePrefixes(); + + for (string prefix: prefixes) + providers.insert({ prefix, provider }); +} + +std::vector +getCredentials(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context, + Json::Value extraData) +{ + std::unique_lock guard(providersLock); + + std::vector result; + + for (auto it = providers.lower_bound(resourceType); it != providers.end(); + ++it) { + if (resourceType.find(it->first) != 0) + break; // not a prefix + auto creds = it->second->getSync(resourceType, resource, context, + extraData); + result.insert(result.end(), creds.begin(), creds.end()); + } + + return result; +} + +Credential +getCredential(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context, + Json::Value extraData, + TimePeriod validTime) +{ + std::unique_lock guard(providersLock); + + cerr << "getCredential" << endl; + + for (auto it = providers.begin(), end = providers.end(); + it != end; ++it) { + cerr << "testing " << it->first << " against " << resourceType + << " " << resource << endl; + if (resourceType.find(it->first) != 0) + break; // not a prefix + cerr << "FOUND" << endl; + + auto creds = it->second->getSync(resourceType, resource, context, + extraData); + if (!creds.empty()) { + cerr << "credentials for " << resourceType << " " << resource + << " are " << endl << jsonEncode(creds[0]) << endl; + return creds[0]; + } + } + + for (auto it = providers.lower_bound(resourceType); it != providers.end(); + ++it) { + cerr << "testing " << it->first << " against " << resourceType + << endl; + if (resourceType.find(it->first) != 0) + break; // not a prefix + cerr << "FOUND" << endl; + + auto creds = it->second->getSync(resourceType, resource, context, + extraData); + if (!creds.empty()) { + cerr << "credentials for " << resourceType << " " << resource + << " are " << endl << jsonEncode(creds[0]) << endl; + return creds[0]; + } + } + + throw ML::Exception("No credentials found for " + resourceType + " " + + resource); +} + + +} // namespace Datacratic + diff --git a/credentials/credential_provider.h b/credentials/credential_provider.h new file mode 100644 index 00000000..a18ee2f5 --- /dev/null +++ b/credentials/credential_provider.h @@ -0,0 +1,41 @@ +/* credential_provider.h -*- C++ -*- + Jeremy Barnes, 5 November 2014 + Copyright (c) 2014 Datacratic Inc. All rights reserved. + + Credential provider structure and registration. +*/ + +#include "soa/credentials/credentials.h" + +#pragma once + +namespace Datacratic { + + +/*****************************************************************************/ +/* CREDENTIAL PROVIDER */ +/*****************************************************************************/ + +/** Base class that can provide credentials to access a given resource. + + Credentials are pluggable to allow for flexible scenarios. +*/ +struct CredentialProvider { + + virtual ~CredentialProvider(); + + virtual std::vector + getResourceTypePrefixes() const = 0; + + virtual std::vector + getSync(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context, + Json::Value extraData) const = 0; + + static void registerProvider(const std::string & name, + std::shared_ptr provider); +}; + + +} // namespace Datacratic diff --git a/credentials/credentials.cc b/credentials/credentials.cc new file mode 100644 index 00000000..4ebd07c2 --- /dev/null +++ b/credentials/credentials.cc @@ -0,0 +1,42 @@ +/** credentials.cc + Jeremy Barnes, 5 November 2014 + Copyright (c) 2014 Datacratic Inc. All rights reserved. + +*/ + +#include "credentials.h" +#include "soa/types/basic_value_descriptions.h" + +using namespace std; + +namespace Datacratic { + +DEFINE_STRUCTURE_DESCRIPTION(Credential); + +CredentialDescription:: +CredentialDescription() +{ + addField("provider", &Credential::provider, + "Provider of credentials"); + addField("protocol", &Credential::protocol, + "Protocol to use to access the service"); + addField("location", &Credential::location, + "Location of the service"); + addField("id", &Credential::id, + "User ID to use to access the service"); + addField("secret", &Credential::secret, + "Secret key to use to access the service"); + addField("extra", &Credential::extra, + "Extra configuration needed to access the service"); + addField("validUntil", &Credential::validUntil, + "Time until which the credential is valid"); +} + +DEFINE_STRUCTURE_DESCRIPTION(CredentialContext); + +CredentialContextDescription:: +CredentialContextDescription() +{ +} + +} // namespace Datacratic diff --git a/credentials/credentials.h b/credentials/credentials.h new file mode 100644 index 00000000..2fcfaad2 --- /dev/null +++ b/credentials/credentials.h @@ -0,0 +1,50 @@ +/* credentials.h -*- C++ -*- + Jeremy Barnes, 5 November 2014 + + A pluggable mechanism for getting credentials. +*/ + +#pragma once + +#include "soa/types/value_description.h" +#include "soa/types/date.h" +#include "soa/types/periodic_utils.h" + +namespace Datacratic { + +struct Credential { + std::string provider; ///< Path through which credential was obtained + std::string protocol; ///< Protocol to use to get to service + std::string location; ///< URI to call to get resource + std::string id; ///< User ID + std::string secret; ///< Password / secret / etc + + Json::Value extra; ///< Other fields + + Date validUntil; +}; + +DECLARE_STRUCTURE_DESCRIPTION(Credential); + +struct CredentialContext { +}; + +DECLARE_STRUCTURE_DESCRIPTION(CredentialContext); + +/** Return credentials for the given resource of the given resource type. + + If none are available, then returns an empty list. +*/ +std::vector +getCredentials(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context, + Json::Value extraData = Json::Value()); + +Credential getCredential(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context = CredentialContext(), + Json::Value extraData = Json::Value(), + TimePeriod validTime = "99999d"); + +} // namespace Datacratic diff --git a/credentials/credentials.mk b/credentials/credentials.mk new file mode 100644 index 00000000..8514e78d --- /dev/null +++ b/credentials/credentials.mk @@ -0,0 +1,11 @@ +# credentials makefile +# Jeremy Barnes, 5 November 2014 +# Copyright (c) 2014 Datacratic Inc. All rights reserved. + +LIBCREDENTIALS_SOURCES := \ + credentials.cc credential_provider.cc + +LIBCREDENTIALS_LINK := \ + arch utils types value_description + +$(eval $(call library,credentials,$(LIBCREDENTIALS_SOURCES),$(LIBCREDENTIALS_LINK))) diff --git a/service/aws.cc b/service/aws.cc index f1f72684..8fa8dd70 100644 --- a/service/aws.cc +++ b/service/aws.cc @@ -23,15 +23,6 @@ using namespace std; using namespace ML; - -namespace { - -std::mutex awsCredentialsLock; -std::map awsCredentials; - -} // file scope - - namespace Datacratic { template @@ -595,32 +586,4 @@ performGet(RestParams && params, resultSelector); } -void registerAwsCredentials(const string & accessKeyId, - const string & accessKey) -{ - unique_lock guard(awsCredentialsLock); - - string & entry = awsCredentials[accessKeyId]; - if (entry.empty()) { - entry = accessKey; - } - else { - if (entry != accessKey) { - throw ML::Exception("access key id '%s' already registered with a" - " different key", accessKeyId.c_str()); - } - } -} - -string getAwsAccessKey(const string & accessKeyId) -{ - auto it = awsCredentials.find(accessKeyId); - if (it == awsCredentials.end()) { - throw ML::Exception("no access key registered for id '%s'", - accessKeyId.c_str()); - } - - return it->second; -} - } // namespace Datacratic diff --git a/service/aws.h b/service/aws.h index f29a8c3e..ef883ea7 100644 --- a/service/aws.h +++ b/service/aws.h @@ -174,12 +174,4 @@ struct AwsBasicApi : public AwsApi { HttpRestProxy proxy; }; -/** Register an AWS access key for future referencing in urls or association - * with buckets */ -void registerAwsCredentials(const std::string & accessKeyId, - const std::string & accessKey); - -/** Returns the key associated with the access key id */ -std::string getAwsAccessKey(const std::string & accessKeyId); - } // namespace Datacratic diff --git a/service/named_endpoint.cc b/service/named_endpoint.cc index 8e4a310e..d35026e9 100644 --- a/service/named_endpoint.cc +++ b/service/named_endpoint.cc @@ -39,7 +39,9 @@ NamedEndpoint:: publishAddress(const std::string & address, const Json::Value & addressConfig) { - ExcAssert(config); + // If we didn't set up a configuration endpoint or name, then just no-op + if (!config) + return; //cerr << "publishing " << address << " with " << addressConfig << endl; config->setUnique(endpointName + "/" + address, diff --git a/service/remote_credential_provider.cc b/service/remote_credential_provider.cc new file mode 100644 index 00000000..48bb21cb --- /dev/null +++ b/service/remote_credential_provider.cc @@ -0,0 +1,168 @@ +/** remote_credential_provider.cc + Jeremy Barnes, 12 November 2014 + Copyright (c) 2014 Datacratic Inc. All rights reserved. + + Credentials provider that gets credentials from a remote source. +*/ + +#include "remote_credential_provider.h" + + +using namespace std; + + +namespace Datacratic { + +/*****************************************************************************/ +/* CREDENTIALS DAEMON CLIENT */ +/*****************************************************************************/ + +CredentialsDaemonClient:: +CredentialsDaemonClient(const std::string & uri) +{ + connect(uri); +} + +CredentialsDaemonClient:: +~CredentialsDaemonClient() +{ +} + +void +CredentialsDaemonClient:: +connect(const std::string & uri) +{ + if (uri.find("http") != 0) { + throw ML::Exception("'uri' parameter does not start with http: " + uri); + } + conn.init(uri); + + try { + auto resp = conn.get("/info"); + if (resp.code() != 200) { + throw ML::Exception("Failed to connect to the credentials daemon"); + } + } + catch (const std::exception & exc) { + cerr << "Failed to connect to the credentials daemon: " << exc.what() + << endl; + throw; + } +} + +std::vector +CredentialsDaemonClient:: +getCredentials(const std::string & resourceType, + const std::string & resource, + const std::string & role, + const std::string & operation, + const TimePeriod & validity, + const Json::Value & extra) const +{ + cerr << "getCredentials " << resourceType << " " << resource << endl; + + string uri = "/v1/types/" + resourceType + "/resources/" + resource + + "/credentials"; + RestParams params; + if (role != "") + params.emplace_back("role", role); + if (operation != "") + params.emplace_back("operation", operation); + if (validity != TimePeriod()) + params.emplace_back("validity", validity.toString()); + if (!extra.isNull()) + params.emplace_back("extra", extra.toStringNoNewLine()); + + cerr << "calling" << endl; + + auto res = conn.get(uri, params, {}, 5.0 /* seconds */); + + cerr << "res = " << res << endl; + + if (res.code() != 200) + throw ML::Exception("couldn't get credentials: returned code %d", + res.code()); + + return jsonDecodeStr >(res.body()); +} + + +/*****************************************************************************/ +/* REMOTE CREDENTIAL PROVIDER */ +/*****************************************************************************/ + +RemoteCredentialProvider:: +RemoteCredentialProvider(const std::string & uri) +{ + client.connect(uri); +} + +std::vector +RemoteCredentialProvider:: +getResourceTypePrefixes() const +{ + return {""}; // for all prefixes +} + +std::vector +RemoteCredentialProvider:: +getSync(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context, + Json::Value extraData) const +{ + cerr << "Remote getSync for " << resourceType << " " << resource + << endl; + + std::string role = "default"; + std::string operation = "*"; + TimePeriod validity = "10000d"; + + return client.getCredentials(resourceType, resource, role, operation, + validity, extraData); +} + + +/*****************************************************************************/ +/* REGISTRATION */ +/*****************************************************************************/ + +void addRemoteCredentialProvider(const std::string & uri) +{ + cerr << "Registering remote credential provider at " << uri << endl; + + auto provider = std::make_shared(uri); + + CredentialProvider::registerProvider("Remote provider at " + uri, + provider); +} + +void initRemoteCredentialsFromEnvironment() +{ + for (const char * const * p = environ; *p; ++p) { + string var(*p); + if (var.find("REMOTE_CREDENTIAL_PROVIDER") == 0) { + auto equalPos = var.find('='); + if (equalPos == string::npos) + continue; // strange, no equals + string val(var, equalPos + 1); + + addRemoteCredentialProvider(val); + } + } +} + +namespace { + +struct AtInit { + + AtInit() + { + initRemoteCredentialsFromEnvironment(); + } + +} atInit; + +} // file scope + +} // namespace Datacratic diff --git a/service/remote_credential_provider.h b/service/remote_credential_provider.h new file mode 100644 index 00000000..49b3cf0d --- /dev/null +++ b/service/remote_credential_provider.h @@ -0,0 +1,91 @@ +/** remote_credential_provider.cc + Jeremy Barnes, 12 November 2014 + Copyright (c) 2014 Datacratic Inc. All rights reserved. + + Credentials provider that gets credentials from a remote source. +*/ + + +#pragma once + +#include "soa/credentials/credentials.h" +#include "soa/credentials/credential_provider.h" +#include "soa/service/http_rest_proxy.h" +#include "soa/credentials/credentials.h" +#include "soa/types/periodic_utils.h" + +namespace Datacratic { + + +/*****************************************************************************/ +/* CREDENTIALS DAEMON CLIENT */ +/*****************************************************************************/ + +/** A client for a credentials daemon. + + By writing a daemon, it is possible to implement logic like short-lived + credentials. + + The daemon must respond to the following routes: + 1. GET /interfaces/credentialsd + 2. GET /v1/types//resources//credentials?role=...&operation=...&validity=...&extra=... + +*/ + +struct CredentialsDaemonClient { + CredentialsDaemonClient() = default; + CredentialsDaemonClient(const std::string & uri); + + ~CredentialsDaemonClient(); + + void connect(const std::string & uri); + + std::vector + getCredentials(const std::string & resourceType, + const std::string & resource, + const std::string & role, + const std::string & operation = "*", + const TimePeriod & validity = "1h", + const Json::Value & extra = Json::Value()) const; + +protected: + HttpRestProxy conn; +}; + + +/*****************************************************************************/ +/* REMOTE CREDENTIAL PROVIDER */ +/*****************************************************************************/ + +struct RemoteCredentialProvider: public CredentialProvider { + + RemoteCredentialProvider(const std::string & uri); + + CredentialsDaemonClient client; + + virtual std::vector + getResourceTypePrefixes() const; + + virtual std::vector + getSync(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context, + Json::Value extraData) const; + +}; + +/*****************************************************************************/ +/* REGISTRATION */ +/*****************************************************************************/ + +/** Add the ability to get remote credentials from the given URI. */ +void addRemoteCredentialProvider(const std::string & uri); + +/** This function will scan the environment looking for any entries that + start with REMOTE_CREDENTIAL_PROVIDER, and for each of them will set + up the given URI as a place to look for remote credentials. +*/ +void initRemoteCredentialsFromEnvironment(); + +} // namespace Datacratic + diff --git a/service/rest_request_params.h b/service/rest_request_params.h index d5265713..be5e2396 100644 --- a/service/rest_request_params.h +++ b/service/rest_request_params.h @@ -127,7 +127,7 @@ struct RestParamDefault { const std::string & description, T defaultValue = T()) : name(name), description(description), defaultValue(defaultValue), - defaultValueStr(boost::lexical_cast(defaultValue)) + defaultValueStr(Codec::encode(defaultValue)) { //std::cerr << "created RestParam with " << name << " at " // << this << std::endl; diff --git a/service/rest_request_router.cc b/service/rest_request_router.cc index 464a941a..e2a010da 100644 --- a/service/rest_request_router.cc +++ b/service/rest_request_router.cc @@ -8,6 +8,7 @@ #include "jml/utils/vector_utils.h" #include "jml/arch/exception_handler.h" #include "jml/utils/set_utils.h" +#include "jml/utils/environment.h" using namespace std; @@ -110,6 +111,12 @@ static std::string getVerbsStr(const std::set & verbs) return verbsStr; } +namespace { + +ML::Env_Option TRACE_REST_REQUESTS("TRACE_REST_REQUESTS", false); + +} // file scope + RestRequestRouter:: MatchResult RestRequestRouter:: @@ -117,7 +124,7 @@ processRequest(const RestServiceEndpoint::ConnectionId & connection, const RestRequest & request, RestRequestParsingContext & context) const { - bool debug = false; + bool debug = TRACE_REST_REQUESTS; if (debug) { cerr << "processing request " << request diff --git a/service/s3.cc b/service/s3.cc index 1f6b3107..c1a77209 100644 --- a/service/s3.cc +++ b/service/s3.cc @@ -21,6 +21,8 @@ #include "jml/utils/file_functions.h" #include "jml/utils/info.h" #include "xml_helpers.h" +#include "soa/credentials/credentials.h" +#include "soa/credentials/credential_provider.h" #define CRYPTOPP_ENABLE_NAMESPACE_WEAK 1 #include "crypto++/sha.h" @@ -56,14 +58,14 @@ struct S3UrlFsHandler : public UrlFsHandler { virtual FsObjectInfo getInfo(const Url & url) const { string bucket = url.host(); - auto api = getS3ApiForBucket(bucket); + auto api = getS3ApiForUri(url.toString()); return api->getObjectInfo(bucket, url.path().substr(1)); } virtual FsObjectInfo tryGetInfo(const Url & url) const { string bucket = url.host(); - auto api = getS3ApiForBucket(bucket); + auto api = getS3ApiForUri(url.toString()); return api->tryGetObjectInfo(bucket, url.path().substr(1)); } @@ -74,7 +76,7 @@ struct S3UrlFsHandler : public UrlFsHandler { virtual bool erase(const Url & url, bool throwException) const { string bucket = url.host(); - auto api = getS3ApiForBucket(bucket); + auto api = getS3ApiForUri(url.toString()); if (throwException) { api->eraseObject(bucket, url.path()); return true; @@ -91,7 +93,7 @@ struct S3UrlFsHandler : public UrlFsHandler { const std::string & startAt) const { string bucket = prefix.host(); - auto api = getS3ApiForBucket(bucket); + auto api = getS3ApiForUri(prefix.toString()); bool result = true; @@ -206,23 +208,6 @@ init(const std::string & accessKeyId, this->bandwidthToServiceMbps = bandwidthToServiceMbps; } -void -S3Api::init() -{ - string keyId, key; - std::tie(keyId, key, std::ignore) - = getS3CredentialsFromEnvVar(); - - if (keyId == "" || key == "") { - tie(keyId, key, std::ignore, std::ignore, std::ignore) - = getCloudCredentials(); - } - if (keyId == "" || key == "") - throw ML::Exception("Cannot init S3 API with no keys, environment or creedentials file"); - - this->init(keyId, key); -} - S3Api::Content:: Content(const tinyxml2::XMLDocument & xml) { @@ -452,10 +437,13 @@ performSync() const ::fprintf(stderr, "%s\n", message.c_str()); /* retry on 50X range errors (recoverable) */ - if (responseCode >= 500 and responseCode < 505) { + if (responseCode >= 500 && responseCode < 505) { continue; } else { + cerr << "Unrecoverable S3 error: code " << responseCode + << endl; + cerr << string(body, 0, 4096) << endl; throw ML::Exception("S3 error is unrecoverable"); } } @@ -2278,81 +2266,6 @@ parseUri(const std::string & uri) return make_pair(bucket, object); } -void -S3Handle:: -initS3(const std::string & accessKeyId, - const std::string & accessKey, - const std::string & uriPrefix) -{ - s3.init(accessKeyId, accessKey); - this->s3UriPrefix = uriPrefix; -} - -size_t -S3Handle:: -getS3Buffer(const std::string & filename, char** outBuffer){ - - if (this->s3UriPrefix == "") { - // not initialized; use defaults - string bucket = S3Api::parseUri(filename).first; - auto api = getS3ApiForBucket(bucket); - size_t size = api->getObjectInfo(filename).size; - - // cerr << "size = " << size << endl; - - // TODO: outBuffer exception safety - *outBuffer = new char[size]; - - uint64_t done = 0; - - auto onChunk = [&] (const char * data, - size_t chunkSize, - int chunkIndex, - uint64_t offset, - uint64_t totalSize) - { - ExcAssertEqual(size, totalSize); - ExcAssertLessEqual(offset + chunkSize, totalSize); - std::copy(data, data + chunkSize, *outBuffer + offset); - ML::atomic_add(done, chunkSize); - }; - - api->download(filename, onChunk); - - return size; - } - - auto stats = s3.getObjectInfo(filename); - if (!stats) - throw ML::Exception("unknown s3 object"); - - *outBuffer = new char[stats.size]; - - uint64_t done = 0; - - auto onChunk = [&] (const char * data, - size_t size, - int chunkIndex, - uint64_t offset, - uint64_t totalSize) - { - ExcAssertEqual(stats.size, totalSize); - ExcAssertLessEqual(offset + size, totalSize); - std::copy(data, data + size, *outBuffer + offset); - ML::atomic_add(done, size); - }; - - s3.download(filename, onChunk); - - ExcAssertEqual(done, stats.size); - - // cerr << "done downloading " << stats.size << " bytes from " - // << filename << endl; - - return stats.size; - -} - bool S3Api:: forEachBucket(const OnBucket & onBucket) const @@ -2432,64 +2345,83 @@ getDefaultRedundancy() return defaultRedundancy; } -namespace { +/** getEnv, but compatible with std::string. Returns null string if not + found. +*/ +static std::string getEnv(const char * varName) +{ + const char * val = getenv(varName); + return val ? val : ""; +} -struct S3BucketInfo { - std::string s3Bucket; - std::shared_ptr api; //< Used to access this uri -}; +/** Provider of S3 credentials that are added explicitly for a subset of + buckets. +*/ -std::recursive_mutex s3BucketsLock; -std::unordered_map s3Buckets; +struct S3ExplicitCredentialProvider: public CredentialProvider { -} // file scope + std::vector buckets; + Credential cred; -/** S3 support for filter_ostream opens. Register the bucket name here, and - you can open it directly from s3. -*/ + S3ExplicitCredentialProvider() + { + } -void registerS3Bucket(const std::string & bucketName, - const std::string & accessKeyId, - const std::string & accessKey, - double bandwidthToServiceMbps, - const std::string & protocol, - const std::string & serviceUri) -{ - std::unique_lock guard(s3BucketsLock); - - auto it = s3Buckets.find(bucketName); - if(it != s3Buckets.end()){ - shared_ptr api = it->second.api; - //if the info is different, raise an exception, otherwise return - if (api->accessKeyId != accessKeyId - || api->accessKey != accessKey - || api->bandwidthToServiceMbps != bandwidthToServiceMbps - || api->defaultProtocol != protocol - || api->serviceUri != serviceUri) - { - return; - throw ML::Exception("Trying to re-register a bucket with different " - "parameters"); - } - return; + S3ExplicitCredentialProvider(std::string provider, + std::string id, + std::string secret, + std::vector buckets, + double bandwidthToServiceMbps, + const std::string & protocol, + const std::string & serviceUri) + { + init(provider, id, secret, buckets, bandwidthToServiceMbps, protocol, serviceUri); + } + + void init(std::string provider, + std::string id, + std::string secret, + std::vector buckets, + double bandwidthToServiceMbps = S3Api::defaultBandwidthToServiceMbps, + const std::string & protocol = "http", + const std::string & serviceUri = "s3.amazonaws.com") + { + cred.provider = provider; + cred.protocol = protocol; + cred.location = serviceUri; + cred.id = id; + cred.secret = secret; + cred.extra["bandwithToServiceMbps"] = bandwidthToServiceMbps; + this->buckets = buckets; } - S3BucketInfo info; - info.s3Bucket = bucketName; - info.api = std::make_shared(accessKeyId, accessKey, - bandwidthToServiceMbps, - protocol, serviceUri); - info.api->getEscaped("", "/" + bucketName + "/", - S3Api::Range::Full); //throws if !accessible - s3Buckets[bucketName] = info; - if (accessKeyId.size() > 0 && accessKey.size() > 0) { - registerAwsCredentials(accessKeyId, accessKey); + virtual std::vector + getResourceTypePrefixes() const + { + return { "aws:s3" }; } -} -/** Register S3 with the filter streams API so that a filter_stream can be used to - treat an S3 object as a simple stream. + virtual std::vector + getSync(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context, + Json::Value extraData) const + { + string bucket = S3Api::parseUri(resource).first; + if (!buckets.empty() + && (std::find(buckets.begin(), buckets.end(), bucket) + == buckets.end())) + return {}; + + return { cred }; + } +}; + +bool disableCloudCredentials = false; + +/** Register S3 with the filter streams API so that a filter_stream can be + used to treat an S3 object as a simple stream. */ struct RegisterS3Handler { static std::pair @@ -2559,176 +2491,159 @@ struct RegisterS3Handler { else throw ML::Exception("no way to create s3 handler for non in/out"); } - void registerBuckets() - { - } - - RegisterS3Handler() - { - ML::registerUriHandler("s3", getS3Handler); - } + /** Parse the ~/.cloud_credentials file and add those buckets in. + + The format of that file is as follows: + 1. One entry per line + 2. Tab separated + 3. Comments are '#' in the first position + 4. First entry is the name of the URI scheme (here, s3) + 5. Second entry is the "version" of the configuration (here, 1) + for forward compatibility + 6. The rest of the entries depend upon the scheme; for s3 they are + tab-separated and include the following: + - Access key ID + - Access key + - Bandwidth from this machine to the server (MBPS) + - Protocol (http) + - S3 machine host name (s3.amazonaws.com) + */ + struct CloudCredentialProvider: public CredentialProvider { + + CloudCredentialProvider() + { + // Parse cloud credentials file + + string filename = ""; + const char * home = getenv("HOME"); + if (home != NULL) + filename = home + string("/.cloud_credentials"); + if (filename != "" && ML::fileExists(filename)) { + std::ifstream stream(filename.c_str()); + int lineNum = 1; + for (; stream; ++lineNum) { + string line; + + getline(stream, line); + if (line.empty() || line[0] == '#') + continue; + if (line.find("s3") != 0) + continue; + + vector fields = ML::split(line, '\t'); + + if (fields[0] != "s3") + continue; + + if (fields.size() < 4) { + cerr << "warning: skipping invalid line in ~/.cloud_credentials: " + << line << endl; + continue; + } + + fields.resize(7); + + string version = fields[1]; + if (version != "1") { + cerr << "warning: ignoring unknown version " + << version << " in ~/.cloud_credentials: " + << line << endl; + continue; + } + + string keyId = fields[2]; + string key = fields[3]; + string bandwidth = fields[4]; + string protocol = fields[5]; + string serviceUri = fields[6]; + + double bw = S3Api::defaultBandwidthToServiceMbps; + if (bandwidth != "") + bw = boost::lexical_cast(bandwidth); + if (protocol == "") + protocol = "http"; + if (serviceUri == "") + serviceUri = "s3.amazonaws.com"; + + Credential cred; + cred.provider = "S3CloudCredentials " + filename + ":" + std::to_string(lineNum); + cred.id = keyId; + cred.secret = key; + cred.protocol = protocol; + cred.location = serviceUri; + cred.extra["bandwidthToServiceMbps"] = bw; + + creds.push_back(cred); + } + } + } -} registerS3Handler; + std::vector creds; -bool defaultBucketsRegistered = false; -std::mutex registerBucketsMutex; + virtual std::vector + getResourceTypePrefixes() const + { + return { "aws:s3" }; + } -tuple getCloudCredentials() -{ - string filename = ""; - char* home; - home = getenv("HOME"); - if (home != NULL) - filename = home + string("/.cloud_credentials"); - if (filename != "" && ML::fileExists(filename)) { - std::ifstream stream(filename.c_str()); - while (stream) { - string line; - - getline(stream, line); - if (line.empty() || line[0] == '#') - continue; - if (line.find("s3") != 0) - continue; + virtual std::vector + getSync(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context, + Json::Value extraData) const + { + if (disableCloudCredentials) + return {}; + return creds; + } + }; - vector fields = ML::split(line, '\t'); + struct S3EnvironmentCredentialProvider: public S3ExplicitCredentialProvider { - if (fields[0] != "s3") - continue; + std::vector buckets; + Credential cred; - if (fields.size() < 4) { - cerr << "warning: skipping invalid line in ~/.cloud_credentials: " - << line << endl; - continue; - } - - fields.resize(7); + S3EnvironmentCredentialProvider() + { + init("S3EnvironmentCredentialProvider", + getEnv("S3_KEY_ID"), + getEnv("S3_KEY"), + ML::split(getEnv("S3_BUCKETS"), ',')); + } + }; - string version = fields[1]; - if (version != "1") { - cerr << "warning: ignoring unknown version " - << version << " in ~/.cloud_credentials: " - << line << endl; - continue; - } - - string keyId = fields[2]; - string key = fields[3]; - string bandwidth = fields[4]; - string protocol = fields[5]; - string serviceUri = fields[6]; + RegisterS3Handler() + { + ML::registerUriHandler("s3", getS3Handler); + CredentialProvider::registerProvider + ("s3CloudCredentials", + std::make_shared()); - return make_tuple(keyId, key, bandwidth, protocol, serviceUri); - } + if (getenv("S3_KEY_ID")) + CredentialProvider::registerProvider + ("s3FromEnvironment", + std::make_shared()); } - return make_tuple("", "", "", "", ""); -} - -std::string getEnv(const char * varName) -{ - const char * val = getenv(varName); - return val ? val : ""; -} -tuple > -getS3CredentialsFromEnvVar() -{ - return make_tuple(getEnv("S3_KEY_ID"), getEnv("S3_KEY"), - ML::split(getEnv("S3_BUCKETS"), ',')); -} +} registerS3Handler; -/** Parse the ~/.cloud_credentials file and add those buckets in. - - The format of that file is as follows: - 1. One entry per line - 2. Tab separated - 3. Comments are '#' in the first position - 4. First entry is the name of the URI scheme (here, s3) - 5. Second entry is the "version" of the configuration (here, 1) - for forward compatibility - 6. The rest of the entries depend upon the scheme; for s3 they are - tab-separated and include the following: - - Access key ID - - Access key - - Bandwidth from this machine to the server (MBPS) - - Protocol (http) - - S3 machine host name (s3.amazonaws.com) - - If S3_KEY_ID and S3_KEY environment variables are specified, - they will be used first. +/** S3 support for filter_ostream opens. Register the bucket name here, and + you can open it directly from s3. */ -void registerDefaultBuckets() -{ - if (defaultBucketsRegistered) - return; - - std::unique_lock guard(registerBucketsMutex); - defaultBucketsRegistered = true; - - tuple cloudCredentials = - getCloudCredentials(); - if (get<0>(cloudCredentials) != "") { - string keyId = get<0>(cloudCredentials); - string key = get<1>(cloudCredentials); - string bandwidth = get<2>(cloudCredentials); - string protocol = get<3>(cloudCredentials); - string serviceUri = get<4>(cloudCredentials); - - if (protocol == "") - protocol = "http"; - if (bandwidth == "") - bandwidth = "20.0"; - if (serviceUri == "") - serviceUri = "s3.amazonaws.com"; - - registerS3Buckets(keyId, key, boost::lexical_cast(bandwidth), - protocol, serviceUri); - return; - } - string keyId; - string key; - vector buckets; - - std::tie(keyId, key, buckets) = getS3CredentialsFromEnvVar(); - if (keyId != "" && key != "") { - if (buckets.empty()) { - registerS3Buckets(keyId, key); - } - else { - for (string bucket: buckets) - registerS3Bucket(bucket, keyId, key); - } - } - else - cerr << "WARNING: registerDefaultBuckets needs either a " - ".cloud_credentials or S3_KEY_ID and S3_KEY environment " - " variables" << endl; - -#if 0 - char* configFilenameCStr = getenv("CONFIG"); - string configFilename = (configFilenameCStr == NULL ? - string() : - string(configFilenameCStr)); - if(configFilename != "") - { - ML::File_Read_Buffer buf(configFilename); - Json::Value config = Json::parse(string(buf.start(), buf.end())); - if(config.isMember("s3")) - { - registerS3Buckets( - config["s3"]["accessKeyId"].asString(), - config["s3"]["accessKey"].asString(), - 20., - "http", - "s3.amazonaws.com"); - return; - } - } - cerr << "WARNING: registerDefaultBuckets needs either a .cloud_credentials" - " file or an environment variable CONFIG pointing toward a file " - "having keys s3.accessKey and s3.accessKeyId" << endl; -#endif +void registerS3Bucket(const std::string & bucketName, + const std::string & accessKeyId, + const std::string & accessKey, + double bandwidthToServiceMbps, + const std::string & protocol, + const std::string & serviceUri) +{ + CredentialProvider::registerProvider + ("s3UserRegisteredBucket", + std::make_shared + ("registerS3Bucket()", accessKeyId, accessKey, + vector({ bucketName }), + bandwidthToServiceMbps, protocol, serviceUri)); } void registerS3Buckets(const std::string & accessKeyId, @@ -2737,68 +2652,45 @@ void registerS3Buckets(const std::string & accessKeyId, const std::string & protocol, const std::string & serviceUri) { - std::unique_lock guard(s3BucketsLock); - int bucketCount(0); auto api = std::make_shared(accessKeyId, accessKey, bandwidthToServiceMbps, protocol, serviceUri); + vector bucketNames; + auto onBucket = [&] (const std::string & bucketName) { - //cerr << "got bucket " << bucketName << endl; - - S3BucketInfo info; - info.s3Bucket = bucketName; - info.api = api; - s3Buckets[bucketName] = info; - bucketCount++; - + bucketNames.push_back(bucketName); return true; }; api->forEachBucket(onBucket); - if (bucketCount == 0) { - cerr << "registerS3Buckets: no bucket registered\n"; + if (bucketNames.empty()) { + cerr << "warning: no bucket names registered"; + } else { + CredentialProvider::registerProvider + ("s3UserRegisteredBucket", + std::make_shared + ("registerS3Buckets()", accessKeyId, accessKey, + bucketNames, bandwidthToServiceMbps, protocol, serviceUri)); } } -std::shared_ptr getS3ApiForBucket(const std::string & bucketName) +/** Extract the bandwidth to the service from a credentials object. */ +static double getBandwidth(const Credential & cred) { - std::unique_lock guard(s3BucketsLock); - auto it = s3Buckets.find(bucketName); - if (it == s3Buckets.end()) { - // On demand, load up the configuration file before we fail - registerDefaultBuckets(); - it = s3Buckets.find(bucketName); - } - if (it == s3Buckets.end()) { - throw ML::Exception("unregistered s3 bucket " + bucketName); - } - return it->second.api; + if (cred.extra.isMember("bandwidthToServiceMbps")) + return cred.extra["bandwidthToServiceMbps"].asDouble(); + else return S3Api::defaultBandwidthToServiceMbps; } std::shared_ptr getS3ApiForUri(const std::string & uri) { - Url url(uri); - - string bucketName = url.host(); - string accessKeyId = url.username(); - if (accessKeyId.empty()) { - return getS3ApiForBucket(bucketName); - } - - string accessKey = url.password(); - if (accessKey.empty()) { - accessKey = getAwsAccessKey(accessKeyId); - } - - auto api = make_shared(accessKeyId, accessKey); - api->getEscaped("", "/" + bucketName + "/", - S3Api::Range::Full); //throws if !accessible - - return api; + // Get the credentials + auto creds = getCredential("aws:s3", uri); + return std::make_shared(creds.id, creds.secret, getBandwidth(creds), + creds.protocol, creds.location); } - } // namespace Datacratic diff --git a/service/s3.h b/service/s3.h index 03f901b3..60cfb117 100644 --- a/service/s3.h +++ b/service/s3.h @@ -77,8 +77,12 @@ struct S3Api : public AwsApi { const std::string & defaultProtocol = "http", const std::string & serviceUri = "s3.amazonaws.com"); + /** Set up the API, getting its credentials from the default credentials + handler. + */ + //void init(); + /** Set up the API to called with the given credentials. */ - void init(); void init(const std::string & accessKeyId, const std::string & accessKey, double bandwidthToServiceMbps = defaultBandwidthToServiceMbps, @@ -632,17 +636,6 @@ struct S3Api : public AwsApi { }; -struct S3Handle{ - S3Api s3; - std::string s3UriPrefix; - - void initS3(const std::string & accessKeyId, - const std::string & accessKey, - const std::string & uriPrefix); - - size_t getS3Buffer(const std::string & filename, char** outBuffer); -}; - /** S3 support for filter_ostream opens. Register the bucket name here, and you can open it directly from s3. */ @@ -665,21 +658,12 @@ void registerS3Buckets(const std::string & accessKeyId, const std::string & protocol = "http", const std::string & serviceUri = "s3.amazonaws.com"); -std::shared_ptr getS3ApiForBucket(const std::string & bucketName); - -std::shared_ptr getS3ApiForUri(const std::string & uri); - -std::tuple - getCloudCredentials(); - -/** Returns the keyId, key and list of buckets to register (can be empty, - which means all) from the environment variables - - S3_KEY_ID, S3_KEY and S3_BUCKETS +/** Returns an S3Api constructed to access the given URI. Will look up its + own credentials using registered credential providers, or one which was + registered using registerS3Bucket or registerS3Buckets, or + ~/.cloud_credentials, or S3_KEY_ID, S3_KEY and S3_BUCKETS environment + variables. */ -std::tuple > -getS3CredentialsFromEnvVar(); - -// std::pair getDefaultCredentials(); +std::shared_ptr getS3ApiForUri(const std::string & uri); } // namespace Datacratic diff --git a/service/service.mk b/service/service.mk index 65ca80f7..28f5a7e6 100644 --- a/service/service.mk +++ b/service/service.mk @@ -54,9 +54,10 @@ LIBSERVICES_SOURCES := \ http_rest_proxy.cc \ xml_helpers.cc \ nprobe.cc \ - logs.cc + logs.cc \ + remote_credential_provider.cc -LIBSERVICES_LINK := opstats curl curlpp boost_regex zeromq zookeeper_mt ACE arch utils jsoncpp boost_thread zmq types tinyxml2 boost_system value_description +LIBSERVICES_LINK := opstats curl curlpp boost_regex zeromq zookeeper_mt ACE arch utils jsoncpp boost_thread zmq types tinyxml2 boost_system value_description credentials $(eval $(call library,services,$(LIBSERVICES_SOURCES),$(LIBSERVICES_LINK))) diff --git a/service/zmq_named_pub_sub.h b/service/zmq_named_pub_sub.h index a36b0d02..c8864f3b 100644 --- a/service/zmq_named_pub_sub.h +++ b/service/zmq_named_pub_sub.h @@ -11,6 +11,7 @@ #include "typed_message_channel.h" #include #include "jml/arch/backtrace.h" +#include namespace Datacratic { diff --git a/soa.mk b/soa.mk index a4f195f8..28cdb56b 100644 --- a/soa.mk +++ b/soa.mk @@ -6,6 +6,7 @@ $(eval $(call include_sub_make,js)) $(eval $(call include_sub_make,sync)) $(eval $(call include_sub_make,sigslot)) $(eval $(call include_sub_make,gc)) +$(eval $(call include_sub_make,credentials)) $(eval $(call include_sub_make,service)) $(eval $(call include_sub_make,logger)) $(eval $(call include_sub_make,launcher)) diff --git a/types/url.cc b/types/url.cc index 66887db1..13885db4 100644 --- a/types/url.cc +++ b/types/url.cc @@ -181,8 +181,12 @@ std::string Url:: path() const { - if (url->scheme() == "file") - return url->host() + url->path(); + if (url->scheme() == "file") { + return string(original, 7); // truncate "file://" + if (url->path() != "/") + return url->host() + url->path(); + else return url->host(); + } else return url->path(); } diff --git a/types/value_description.h b/types/value_description.h index b0a04030..72ed3c78 100644 --- a/types/value_description.h +++ b/types/value_description.h @@ -335,28 +335,6 @@ struct RegisterValueDescriptionI { static const RegisterValueDescription registerValueDescription##type; \ } -/*****************************************************************************/ -/* PURE VALUE DESCRIPTION */ -/*****************************************************************************/ - -template -struct PureValueDescription : public ValueDescription { - PureValueDescription() : - ValueDescription(ValueKind::ATOM, &typeid(T)) { - } - - virtual void parseJson(void * val, JsonParsingContext & context) const {}; - virtual void printJson(const void * val, JsonPrintingContext & context) const {}; - virtual bool isDefault(const void * val) const { return false; } - virtual void setDefault(void * val) const {} - virtual void copyValue(const void * from, void * to) const {} - virtual void moveValue(void * from, void * to) const {} - virtual void swapValues(void * from, void * to) const {} - virtual void * constructDefault() const {return nullptr;} - virtual void destroy(void *) const {} - -}; - /*****************************************************************************/ /* VALUE DESCRIPTION TEMPLATE */ /*****************************************************************************/ @@ -648,6 +626,28 @@ maybeGetDefaultDescriptionShared(T * = 0) return result; } +/*****************************************************************************/ +/* PURE VALUE DESCRIPTION */ +/*****************************************************************************/ + +template +struct PureValueDescription : public ValueDescriptionT { + PureValueDescription() : + ValueDescriptionT(ValueKind::ATOM) { + } + + virtual void parseJson(void * val, JsonParsingContext & context) const {}; + virtual void printJson(const void * val, JsonPrintingContext & context) const {}; + virtual bool isDefault(const void * val) const { return false; } + virtual void setDefault(void * val) const {} + virtual void copyValue(const void * from, void * to) const {} + virtual void moveValue(void * from, void * to) const {} + virtual void swapValues(void * from, void * to) const {} + virtual void * constructDefault() const {return nullptr;} + virtual void destroy(void *) const {} + +}; + /*****************************************************************************/ /* VALUE DESCRIPTION CONCRETE IMPL */ @@ -2028,3 +2028,9 @@ inline Json::Value jsonEncode(const char * str) #define CREATE_ENUM_DESCRIPTION(Type) \ CREATE_ENUM_DESCRIPTION_NAMED(Type##Description, Type) + +#define HAS_NO_VALUE_DESCRIPTION(Type) \ + inline PureValueDescription * getDefaultDescription(Type *) \ + { \ + return new PureValueDescription(); \ + }