From c599134dbb9521da02799eecfffea6ce43dbd7cb Mon Sep 17 00:00:00 2001 From: Jeremy Barnes Date: Sat, 14 Jun 2014 16:21:22 -0400 Subject: [PATCH 01/15] Allow chunked encoding, incremental sending of body on HTTP REST calls (PLAT-562) --- service/http_endpoint.cc | 11 +++--- service/http_endpoint.h | 22 ++++++++++-- service/http_named_endpoint.cc | 20 +++++++++++ service/http_named_endpoint.h | 4 +++ service/rest_service_endpoint.cc | 60 ++++++++++++++++++++++++++++++++ 5 files changed, 111 insertions(+), 6 deletions(-) diff --git a/service/http_endpoint.cc b/service/http_endpoint.cc index e6a14e2a..61292038 100644 --- a/service/http_endpoint.cc +++ b/service/http_endpoint.cc @@ -363,10 +363,13 @@ putResponseOnWire(HttpResponse response, responseStr.append(response.contentType); responseStr.append("\r\n"); } - responseStr.append("Content-Length: "); - responseStr.append(to_string(response.body.length())); - responseStr.append("\r\n"); - responseStr.append("Connection: Keep-Alive\r\n"); + + if (response.sendBody) { + responseStr.append("Content-Length: "); + responseStr.append(to_string(response.body.length())); + responseStr.append("\r\n"); + responseStr.append("Connection: Keep-Alive\r\n"); + } for (auto & h: response.extraHeaders) { responseStr.append(h.first); diff --git a/service/http_endpoint.h b/service/http_endpoint.h index 7e46b7c4..6f3910b8 100644 --- a/service/http_endpoint.h +++ b/service/http_endpoint.h @@ -37,7 +37,23 @@ struct HttpResponse { responseStatus(getResponseReasonPhrase(responseCode)), contentType(contentType), body(body), - extraHeaders(extraHeaders) + extraHeaders(extraHeaders), + sendBody(true) + { + } + + /** Construct an HTTP response header only, with no body. No content- + length will be inferred. */ + + HttpResponse(int responseCode, + std::string contentType, + std::vector > extraHeaders + = std::vector >()) + : responseCode(responseCode), + responseStatus(getResponseReasonPhrase(responseCode)), + contentType(contentType), + extraHeaders(extraHeaders), + sendBody(false) { } @@ -49,7 +65,8 @@ struct HttpResponse { responseStatus(getResponseReasonPhrase(responseCode)), contentType("application/json"), body(boost::trim_copy(body.toString())), - extraHeaders(extraHeaders) + extraHeaders(extraHeaders), + sendBody(true) { } @@ -58,6 +75,7 @@ struct HttpResponse { std::string contentType; std::string body; std::vector > extraHeaders; + bool sendBody; }; diff --git a/service/http_named_endpoint.cc b/service/http_named_endpoint.cc index 5c47ec2c..9b821432 100644 --- a/service/http_named_endpoint.cc +++ b/service/http_named_endpoint.cc @@ -217,6 +217,8 @@ sendResponse(int code, const std::string & contentType, RestParams headers) { + // Recycle back to a new handler once done so that the next connection can be + // handled. auto onSendFinished = [=] { this->transport().associateWhenHandlerFinished (std::make_shared(endpoint), @@ -230,6 +232,24 @@ sendResponse(int code, onSendFinished); } +void +HttpNamedEndpoint::RestConnectionHandler:: +sendResponseHeader(int code, + const std::string & contentType, + RestParams headers) +{ + auto onSendFinished = [=] { + // Do nothing once we've finished sending the response, so that + // the connection isn't closed + }; + + for (auto & h: endpoint->extraHeaders) + headers.push_back(h); + + putResponseOnWire(HttpResponse(code, contentType, headers), + onSendFinished); +} + /*****************************************************************************/ /* HTTP NAMED REST PROXY */ diff --git a/service/http_named_endpoint.h b/service/http_named_endpoint.h index f7633c8f..d58f0012 100644 --- a/service/http_named_endpoint.h +++ b/service/http_named_endpoint.h @@ -79,6 +79,10 @@ struct HttpNamedEndpoint : public NamedEndpoint, public HttpEndpoint { const std::string & body, const std::string & contentType, RestParams headers = RestParams()); + + void sendResponseHeader(int code, + const std::string & contentType, + RestParams headers = RestParams()); }; typedef std::functionresponseSent = true; } +void +RestServiceEndpoint::ConnectionId:: +sendHttpResponseHeader(int responseCode, + const std::string & contentType, + ssize_t contentLength, + const RestParams & headers_) const +{ + if (itl->responseSent) + throw ML::Exception("response already sent"); + + if (!itl->http) + throw ML::Exception("sendHttpResponseHeader only works on HTTP connections"); + + if (itl->endpoint->logResponse) + itl->endpoint->logResponse(*this, responseCode, "", contentType); + + RestParams headers = headers_; + if (contentLength == CHUNKED_ENCODING) { + itl->chunkedEncoding = true; + headers.push_back({"Transfer-Encoding", "chunked"}); + } + else if (contentLength >= 0) { + headers.push_back({"Content-Length", to_string(contentLength) }); + } + else { + itl->keepAlive = false; + } + + itl->http->sendResponseHeader(responseCode, contentType, headers); +} + +void +RestServiceEndpoint::ConnectionId:: +sendPayload(const std::string & payload) +{ + if (itl->chunkedEncoding) { + if (payload.empty()) { + throw ML::Exception("Can't send empty chunk over a chunked connection"); + } + string length = ML::format("%llx\r\n", (long long)payload.length()); + itl->http->sendHttpChunk(payload, HttpConnectionHandler::NEXT_CONTINUE); + } + else itl->http->send(payload); +} + +void +RestServiceEndpoint::ConnectionId:: +finishResponse() +{ + if (itl->chunkedEncoding) { + itl->http->sendHttpChunk("", HttpConnectionHandler::NEXT_RECYCLE); + } + else if (!itl->keepAlive) { + itl->http->closeConnection(); + } + + itl->responseSent = true; +} + + /*****************************************************************************/ /* REST SERVICE ENDPOINT */ From d154f715edea35990cb3888c90c8413aaf08ac6e Mon Sep 17 00:00:00 2001 From: Jeremy Barnes Date: Wed, 18 Jun 2014 11:08:55 -0400 Subject: [PATCH 02/15] Add ability to turn off log categories by default (PLAT-564) Adds an extra boolean parameter to the log category that allows it to be turned off by default, to support the 'recompile with verbose logging' use-case. This could be combined at a later stage with the ability to modify logging via environment variables or a SHM segment and a signal. --- service/logs.cc | 31 ++++++++++++++----------- service/logs.h | 45 ++++++++++++++++++++++++++++++++---- service/testing/logs_test.cc | 2 ++ 3 files changed, 60 insertions(+), 18 deletions(-) diff --git a/service/logs.cc b/service/logs.cc index 0ea12518..7d2f4ea0 100644 --- a/service/logs.cc +++ b/service/logs.cc @@ -13,7 +13,7 @@ #include #include -using namespace Datacratic; +namespace Datacratic { void Logging::ConsoleWriter::head(char const * timestamp, char const * name, @@ -86,7 +86,7 @@ struct Logging::CategoryData { static CategoryData * getRoot(); static CategoryData * get(char const * name); - static CategoryData * create(char const * name, char const * super); + static CategoryData * create(char const * name, char const * super, bool enabled); static void destroy(CategoryData * name); void activate(bool recurse = true); @@ -95,9 +95,9 @@ struct Logging::CategoryData { private: - CategoryData(char const * name) : + CategoryData(char const * name, bool enabled) : initialized(false), - enabled(true), + enabled(enabled), name(name), parent(nullptr) { } @@ -115,14 +115,14 @@ Logging::CategoryData * Logging::CategoryData::getRoot() { CategoryData * root = get("*"); if (root) return root; - getRegistry().categories["*"].reset(root = new CategoryData("*")); + getRegistry().categories["*"].reset(root = new CategoryData("*", true /* enabled */)); root->parent = root; root->writer = std::make_shared(); return root; } -Logging::CategoryData * Logging::CategoryData::create(char const * name, char const * super) { +Logging::CategoryData * Logging::CategoryData::create(char const * name, char const * super, bool enabled) { Registry& registry = getRegistry(); std::lock_guard guard(registry.lock); @@ -130,7 +130,7 @@ Logging::CategoryData * Logging::CategoryData::create(char const * name, char co CategoryData * data = get(name); if (!data) { - registry.categories[name].reset(data = new CategoryData(name)); + registry.categories[name].reset(data = new CategoryData(name, enabled)); } else { ExcCheck(!data->initialized, @@ -141,18 +141,16 @@ Logging::CategoryData * Logging::CategoryData::create(char const * name, char co data->parent = get(super); if (!data->parent) { - registry.categories[super].reset(data->parent = new CategoryData(super)); + registry.categories[super].reset(data->parent = new CategoryData(super, enabled)); } data->parent->children.push_back(data); if (data->parent->initialized) { data->writer = data->parent->writer; - data->enabled = data->parent->enabled; } else { data->writer = root->writer; - data->enabled = root->enabled; } return data; @@ -222,12 +220,16 @@ Logging::Category::Category(CategoryData * data) : data(data) { } -Logging::Category::Category(char const * name, Category & super) : - data(CategoryData::create(name, super.name())) { +Logging::Category::Category(char const * name, Category & super, bool enabled) : + data(CategoryData::create(name, super.name(), enabled)) { } -Logging::Category::Category(char const * name, char const * super) : - data(CategoryData::create(name, super)) { +Logging::Category::Category(char const * name, char const * super, bool enabled) : + data(CategoryData::create(name, super, enabled)) { +} + +Logging::Category::Category(char const * name, bool enabled) : + data(CategoryData::create(name, "*", enabled)) { } Logging::Category::~Category() @@ -288,3 +290,4 @@ void Logging::Thrower::operator&(std::ostream & stream) { throw ML::Exception(message); } +} // namespace Datacratic diff --git a/service/logs.h b/service/logs.h index 15252d8e..3ef32e9a 100644 --- a/service/logs.h +++ b/service/logs.h @@ -1,4 +1,4 @@ -/* logs.h +/* logs.h -*- C++ -*- Eric Robert, 9 October 2013 Copyright (c) 2013 Datacratic. All rights reserved. @@ -64,8 +64,9 @@ struct Logging struct CategoryData; struct Category { - Category(char const * name, Category & super); - Category(char const * name, char const * super = "*"); + Category(char const * name, Category & super, bool enabled = true); + Category(char const * name, char const * super, bool enabled = true); + Category(char const * name, bool enabled = true); ~Category(); Category(const Category&) = delete; @@ -76,6 +77,26 @@ struct Logging bool isEnabled() const; bool isDisabled() const; + /// Type that is convertible to bool but nothing else for operator bool + typedef void (Category::* boolConvertibleType)() const; + + /** Boolean conversion allows you to know if it's enabled. Usage: + + Logging::Category logMyComponent("myComponent"); + + std::string output; + Json::Value loggingInfo; + + std::tie(output, loggingInfo) + = performCallMaybeWithExpensiveLoggingInfo((bool)logMyComponent); + + LOG(myComponent) << loggingInfo; + */ + operator boolConvertibleType () const + { + return isEnabled() ? &Category::dummy : nullptr; + } + std::shared_ptr const & getWriter() const; void writeTo(std::shared_ptr output, bool recurse = true); @@ -88,8 +109,10 @@ struct Logging private: Category(CategoryData * data); - CategoryData * data; + + // operator bool result + void dummy() const {} }; struct Printer { @@ -157,10 +180,24 @@ struct Logging } // namespace Datacratic +/** Macro to call to log a message to the given group. Usage is as follows: + + Logging::Category errors("errors"); + + LOG(errors) << "error frobbing: " << errorMessage << endl; +*/ #define LOG(group, ...) \ group.isDisabled() ? (void) 0 : Logging::Printer(group) & \ group.beginWrite(__PRETTY_FUNCTION__, __FILE__, __LINE__ __VA_ARGS__) +/** Macro to log a thrown exeption to the given group and then throw it. Usage is + as follows: + + Logging::Category logMyComponent("myComponent"); + + if (badErrorCondition) + THROW(logMyComponent) << "fatal error with bad error condition"; +*/ #define THROW(group, ...) \ Logging::Thrower(group) & \ group.beginWrite(__PRETTY_FUNCTION__, __FILE__, __LINE__ __VA_ARGS__) diff --git a/service/testing/logs_test.cc b/service/testing/logs_test.cc index 19321f5f..256120b0 100644 --- a/service/testing/logs_test.cc +++ b/service/testing/logs_test.cc @@ -24,12 +24,14 @@ BOOST_AUTO_TEST_CASE(blah) { Logging::Category d("d", "a"); Logging::Category e("e", "d"); + Logging::Category f("f", "d", false /* enabled */); BOOST_CHECK(a.isEnabled()); BOOST_CHECK(b.isEnabled()); BOOST_CHECK(c.isEnabled()); BOOST_CHECK(d.isEnabled()); BOOST_CHECK(e.isEnabled()); + BOOST_CHECK(!f.isEnabled()); BOOST_CHECK(!!a.getWriter()); BOOST_CHECK(!!b.getWriter()); From dcf0bef0c90bf192c7a505764fcff6bf6c8b9716 Mon Sep 17 00:00:00 2001 From: Jeremy Barnes Date: Sun, 29 Jun 2014 16:18:09 -0400 Subject: [PATCH 03/15] More flexibility in handling signal errors in Runner (TRIVIAL) This change adds a mustSucceed flag to the kill() and signal() functions in Runner, which allows them to be called unconditionally. This is especially useful in shutdowns, where we don't want an exception if we tried to stop it but it was already stopped. The default behaviour remains the same: throw an exception. They will also return whether they succeded or not. TRIVIAL because semantics of any existing calls have not changed. --- service/runner.cc | 26 +++++++++++++++++--------- service/runner.h | 20 ++++++++++++++++---- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/service/runner.cc b/service/runner.cc index 75ed8744..228a3d0d 100644 --- a/service/runner.cc +++ b/service/runner.cc @@ -465,25 +465,33 @@ run(const vector & command, } } -void +bool Runner:: -kill(int signum) const +kill(int signum, bool mustSucceed) const { - if (childPid_ <= 0) - throw ML::Exception("subprocess not available"); + if (childPid_ <= 0) { + if (mustSucceed) + throw ML::Exception("subprocess not available"); + else return false; + } ::kill(-childPid_, signum); waitTermination(); + return true; } -void +bool Runner:: -signal(int signum) +signal(int signum, bool mustSucceed) { - if (childPid_ <= 0) - throw ML::Exception("subprocess not available"); - + if (childPid_ <= 0) { + if (mustSucceed) + throw ML::Exception("subprocess not available"); + else return false; + } + ::kill(childPid_, signum); + return true; } bool diff --git a/service/runner.h b/service/runner.h index 530d250a..e7d45a85 100644 --- a/service/runner.h +++ b/service/runner.h @@ -114,11 +114,23 @@ struct Runner: public Epoller { const std::shared_ptr & stdErrSink = nullptr); /** Kill the subprocess with the given signal, then wait for it to - terminate. */ - void kill(int signal = SIGTERM) const; + terminate. - /** Send the given signal, but don't wait for it to terminate. */ - void signal(int signum); + If mustSucceed = true, then an exception will be thrown if there + is no process. + + Returns whether or not the call succeeded. + */ + bool kill(int signal = SIGTERM, bool mustSucceed = true) const; + + /** Send the given signal, but don't wait for it to terminate. + + If mustSucceed = true, then an exception will be thrown if there + is no process. + + Returns whether or not the call succeeded. + */ + bool signal(int signum, bool mustSucceed = true); /** Synchronous wait for the subprocess to start. Returns true if the process started, or false if it wasn't able to start. From ede330566658bd0b04a648c6da4a1ad45d1b175a Mon Sep 17 00:00:00 2001 From: Jeremy Barnes Date: Tue, 1 Jul 2014 09:55:53 -0400 Subject: [PATCH 04/15] Make HttpNamedEndpoint keep shared pointer to connection This makes it possible to avoid segmentation faults in the case where the remote end drops the connection. Previously it would lead to the connection object being deleted and a dangling pointer. Mostly relevant to long-running connections that are handled asynchronously. In the synchronous case the connection is locked for the duration and recycled afterwards, so it can't cause a problem. --- service/http_named_endpoint.cc | 13 +++++++++---- service/http_named_endpoint.h | 3 ++- service/rest_service_endpoint.cc | 2 +- service/rest_service_endpoint.h | 6 +++--- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/service/http_named_endpoint.cc b/service/http_named_endpoint.cc index 9b821432..fb8adbb4 100644 --- a/service/http_named_endpoint.cc +++ b/service/http_named_endpoint.cc @@ -136,7 +136,11 @@ std::shared_ptr HttpNamedEndpoint:: makeNewHandler() { - return std::make_shared(this); + auto res = std::make_shared(this); + + // Allow it to get a shared pointer to itself + res->sharedThis = res; + return res; } @@ -156,7 +160,9 @@ handleHttpPayload(const HttpHeader & header, const std::string & payload) { try { - endpoint->onRequest(this, header, payload); + auto th = sharedThis.lock(); + ExcAssert(th); + endpoint->onRequest(th, header, payload); } catch(const std::exception& ex) { Json::Value response; @@ -221,8 +227,7 @@ sendResponse(int code, // handled. auto onSendFinished = [=] { this->transport().associateWhenHandlerFinished - (std::make_shared(endpoint), - "sendResponse"); + (endpoint->makeNewHandler(), "sendResponse"); }; for (auto & h: endpoint->extraHeaders) diff --git a/service/http_named_endpoint.h b/service/http_named_endpoint.h index d58f0012..c5ca8fa2 100644 --- a/service/http_named_endpoint.h +++ b/service/http_named_endpoint.h @@ -61,6 +61,7 @@ struct HttpNamedEndpoint : public NamedEndpoint, public HttpEndpoint { RestConnectionHandler(HttpNamedEndpoint * endpoint); HttpNamedEndpoint * endpoint; + std::weak_ptr sharedThis; virtual void handleHttpPayload(const HttpHeader & header, @@ -85,7 +86,7 @@ struct HttpNamedEndpoint : public NamedEndpoint, public HttpEndpoint { RestParams headers = RestParams()); }; - typedef std::function connection, const HttpHeader & header, const std::string & payload)> OnRequest; diff --git a/service/rest_service_endpoint.cc b/service/rest_service_endpoint.cc index 1bb6e62c..57f4fb4c 100644 --- a/service/rest_service_endpoint.cc +++ b/service/rest_service_endpoint.cc @@ -326,7 +326,7 @@ init(std::shared_ptr config, zmqEndpoint.messageHandler = zmqHandler; httpEndpoint.onRequest - = [=] (HttpNamedEndpoint::RestConnectionHandler * connection, + = [=] (std::shared_ptr connection, const HttpHeader & header, const std::string & payload) { diff --git a/service/rest_service_endpoint.h b/service/rest_service_endpoint.h index f83d249a..94f20668 100644 --- a/service/rest_service_endpoint.h +++ b/service/rest_service_endpoint.h @@ -97,7 +97,7 @@ struct RestServiceEndpoint: public MessageLoop { } /// Initialize for http - ConnectionId(HttpNamedEndpoint::RestConnectionHandler * http, + ConnectionId(std::shared_ptr http, const std::string & requestId, RestServiceEndpoint * endpoint) : itl(new Itl(http, requestId, endpoint)) @@ -105,7 +105,7 @@ struct RestServiceEndpoint: public MessageLoop { } struct Itl { - Itl(HttpNamedEndpoint::RestConnectionHandler * http, + Itl(std::shared_ptr http, const std::string & requestId, RestServiceEndpoint * endpoint) : requestId(requestId), @@ -140,7 +140,7 @@ struct RestServiceEndpoint: public MessageLoop { std::string zmqAddress; std::string requestId; - HttpNamedEndpoint::RestConnectionHandler * http; + std::shared_ptr http; RestServiceEndpoint * endpoint; bool responseSent; Date startDate; From 183fb680b36e46fd481c89580e82bc6c4f92f73c Mon Sep 17 00:00:00 2001 From: Jeremy Barnes Date: Mon, 3 Nov 2014 09:47:11 -0500 Subject: [PATCH 05/15] Added missing header to zmq_named_pub_sub.h (TRIVIAL) --- service/zmq_named_pub_sub.h | 1 + 1 file changed, 1 insertion(+) diff --git a/service/zmq_named_pub_sub.h b/service/zmq_named_pub_sub.h index a36b0d02..c8864f3b 100644 --- a/service/zmq_named_pub_sub.h +++ b/service/zmq_named_pub_sub.h @@ -11,6 +11,7 @@ #include "typed_message_channel.h" #include #include "jml/arch/backtrace.h" +#include namespace Datacratic { From 6cdcde20292165515bfb4c875f9fc7b5119e7617 Mon Sep 17 00:00:00 2001 From: Jeremy Barnes Date: Mon, 3 Nov 2014 10:13:56 -0500 Subject: [PATCH 06/15] Further fixes for file:// URIs to accept relative path (TRIVIAL) --- types/url.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/types/url.cc b/types/url.cc index 66887db1..13885db4 100644 --- a/types/url.cc +++ b/types/url.cc @@ -181,8 +181,12 @@ std::string Url:: path() const { - if (url->scheme() == "file") - return url->host() + url->path(); + if (url->scheme() == "file") { + return string(original, 7); // truncate "file://" + if (url->path() != "/") + return url->host() + url->path(); + else return url->host(); + } else return url->path(); } From b5b91629420697b4643a2e8611f3061d5385365e Mon Sep 17 00:00:00 2001 From: Jeremy Barnes Date: Wed, 5 Nov 2014 16:55:10 -0500 Subject: [PATCH 07/15] Started adding in credentials handling infrastructure --- credentials/credential_provider.cc | 14 +++++++++++ credentials/credential_provider.h | 26 ++++++++++++++++++++ credentials/credentials.cc | 37 ++++++++++++++++++++++++++++ credentials/credentials.h | 39 ++++++++++++++++++++++++++++++ credentials/credentials.mk | 11 +++++++++ service/service.mk | 2 +- soa.mk | 1 + 7 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 credentials/credential_provider.cc create mode 100644 credentials/credential_provider.h create mode 100644 credentials/credentials.cc create mode 100644 credentials/credentials.h create mode 100644 credentials/credentials.mk diff --git a/credentials/credential_provider.cc b/credentials/credential_provider.cc new file mode 100644 index 00000000..7f6eda18 --- /dev/null +++ b/credentials/credential_provider.cc @@ -0,0 +1,14 @@ +/* credential_provider.cc + Jeremy Barnes, 5 November 2014 + Copyright (c) 2014 Datacratic Inc. All rights reserved. + + Basic functionality to get credentials. +*/ + +#include "credential_provider.h" + +namespace Datacratic { + + +} // namespace Datacratic + diff --git a/credentials/credential_provider.h b/credentials/credential_provider.h new file mode 100644 index 00000000..6cc1a522 --- /dev/null +++ b/credentials/credential_provider.h @@ -0,0 +1,26 @@ +/* credential_provider.h -*- C++ -*- + Jeremy Barnes, 5 November 2014 + Copyright (c) 2014 Datacratic Inc. All rights reserved. + + Credential provider structure and registration. +*/ + +#include "soa/credentials/credentials.h" + +#pragma once + +namespace Datacratic { + +struct CredentialProvider { + virtual std::vector + getSync(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context, + Json::Value extraData) const = 0; + + static void registerProvider(const std::string & name, + std::shared_ptr provider); +}; + + +} // namespace Datacratic diff --git a/credentials/credentials.cc b/credentials/credentials.cc new file mode 100644 index 00000000..fcb6d8e1 --- /dev/null +++ b/credentials/credentials.cc @@ -0,0 +1,37 @@ +/** credentials.cc + Jeremy Barnes, 5 November 2014 + Copyright (c) 2014 Datacratic Inc. All rights reserved. + +*/ + +#include "credentials.h" + +using namespace std; + +namespace Datacratic { + +DEFINE_STRUCTURE_DESCRIPTION(Credential); + +CredentialDescription:: +CredentialDescription() +{ +} + +DEFINE_STRUCTURE_DESCRIPTION(CredentialContext); + +CredentialContextDescription:: +CredentialContextDescription() +{ +} + +std::vector +getCredentials(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context, + Json::Value extraData) +{ + return {}; +} + + +} // namespace Datacratic diff --git a/credentials/credentials.h b/credentials/credentials.h new file mode 100644 index 00000000..893fc2f5 --- /dev/null +++ b/credentials/credentials.h @@ -0,0 +1,39 @@ +/* credentials.h -*- C++ -*- + Jeremy Barnes, 5 November 2014 + + A pluggable mechanism for getting credentials. +*/ + +#pragma once + +#include "soa/types/value_description.h" + +namespace Datacratic { + +struct Credential { + std::string id; + std::string secret; + + Json::Value metadata; +}; + +DECLARE_STRUCTURE_DESCRIPTION(Credential); + +struct CredentialContext { + std::string user; + Json::Value userid; +}; + +DECLARE_STRUCTURE_DESCRIPTION(CredentialContext); + +/** Return credentials for the given resource of the given resource type. + + If none are available, then returns an empty list. +*/ +std::vector +getCredentials(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context, + Json::Value extraData); + +} // namespace Datacratic diff --git a/credentials/credentials.mk b/credentials/credentials.mk new file mode 100644 index 00000000..8514e78d --- /dev/null +++ b/credentials/credentials.mk @@ -0,0 +1,11 @@ +# credentials makefile +# Jeremy Barnes, 5 November 2014 +# Copyright (c) 2014 Datacratic Inc. All rights reserved. + +LIBCREDENTIALS_SOURCES := \ + credentials.cc credential_provider.cc + +LIBCREDENTIALS_LINK := \ + arch utils types value_description + +$(eval $(call library,credentials,$(LIBCREDENTIALS_SOURCES),$(LIBCREDENTIALS_LINK))) diff --git a/service/service.mk b/service/service.mk index 65ca80f7..152936c9 100644 --- a/service/service.mk +++ b/service/service.mk @@ -56,7 +56,7 @@ LIBSERVICES_SOURCES := \ nprobe.cc \ logs.cc -LIBSERVICES_LINK := opstats curl curlpp boost_regex zeromq zookeeper_mt ACE arch utils jsoncpp boost_thread zmq types tinyxml2 boost_system value_description +LIBSERVICES_LINK := opstats curl curlpp boost_regex zeromq zookeeper_mt ACE arch utils jsoncpp boost_thread zmq types tinyxml2 boost_system value_description credentials $(eval $(call library,services,$(LIBSERVICES_SOURCES),$(LIBSERVICES_LINK))) diff --git a/soa.mk b/soa.mk index a4f195f8..28cdb56b 100644 --- a/soa.mk +++ b/soa.mk @@ -6,6 +6,7 @@ $(eval $(call include_sub_make,js)) $(eval $(call include_sub_make,sync)) $(eval $(call include_sub_make,sigslot)) $(eval $(call include_sub_make,gc)) +$(eval $(call include_sub_make,credentials)) $(eval $(call include_sub_make,service)) $(eval $(call include_sub_make,logger)) $(eval $(call include_sub_make,launcher)) From 032620de85c83c747aaa3197a560902c20433617 Mon Sep 17 00:00:00 2001 From: Jeremy Barnes Date: Thu, 6 Nov 2014 16:19:00 -0500 Subject: [PATCH 08/15] Refactored S3 credentials to use credential provider --- credentials/credential_provider.cc | 80 ++++++ credentials/credential_provider.h | 17 +- credentials/credentials.cc | 25 +- credentials/credentials.h | 23 +- service/aws.cc | 6 +- service/aws.h | 2 + service/s3.cc | 429 ++++++++++++++++++++--------- service/s3.h | 22 +- 8 files changed, 436 insertions(+), 168 deletions(-) diff --git a/credentials/credential_provider.cc b/credentials/credential_provider.cc index 7f6eda18..75ba679a 100644 --- a/credentials/credential_provider.cc +++ b/credentials/credential_provider.cc @@ -6,9 +6,89 @@ */ #include "credential_provider.h" +#include + +using namespace std; namespace Datacratic { +/*****************************************************************************/ +/* CREDENTIAL PROVIDER */ +/*****************************************************************************/ + +CredentialProvider:: +~CredentialProvider() +{ +} + +namespace { + +std::mutex providersLock; +std::multimap > providers; + +} // file scope + +void +CredentialProvider:: +registerProvider(const std::string & name, + std::shared_ptr provider) +{ + std::unique_lock guard(providersLock); + + auto prefixes = provider->getResourceTypePrefixes(); + + for (string prefix: prefixes) + providers.insert({ prefix, provider }); +} + +std::vector +getCredentials(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context, + Json::Value extraData) +{ + std::unique_lock guard(providersLock); + + std::vector result; + + for (auto it = providers.lower_bound(resourceType); it != providers.end(); + ++it) { + if (resourceType.find(it->first) != 0) + break; // not a prefix + auto creds = it->second->getSync(resourceType, resource, context, + extraData); + result.insert(result.end(), creds.begin(), creds.end()); + } + + return result; +} + +Credential +getCredential(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context, + Json::Value extraData, + TimePeriod validTime) +{ + std::unique_lock guard(providersLock); + + for (auto it = providers.lower_bound(resourceType); it != providers.end(); + ++it) { + if (resourceType.find(it->first) != 0) + break; // not a prefix + auto creds = it->second->getSync(resourceType, resource, context, + extraData); + if (!creds.empty()) { + cerr << "credentials for " << resourceType << " " << resource + << " are " << endl << jsonEncode(creds[0]) << endl; + return creds[0]; + } + } + + throw ML::Exception("No credentials found for " + resourceType + " " + + resource); +} + } // namespace Datacratic diff --git a/credentials/credential_provider.h b/credentials/credential_provider.h index 6cc1a522..a18ee2f5 100644 --- a/credentials/credential_provider.h +++ b/credentials/credential_provider.h @@ -11,13 +11,28 @@ namespace Datacratic { + +/*****************************************************************************/ +/* CREDENTIAL PROVIDER */ +/*****************************************************************************/ + +/** Base class that can provide credentials to access a given resource. + + Credentials are pluggable to allow for flexible scenarios. +*/ struct CredentialProvider { + + virtual ~CredentialProvider(); + + virtual std::vector + getResourceTypePrefixes() const = 0; + virtual std::vector getSync(const std::string & resourceType, const std::string & resource, const CredentialContext & context, Json::Value extraData) const = 0; - + static void registerProvider(const std::string & name, std::shared_ptr provider); }; diff --git a/credentials/credentials.cc b/credentials/credentials.cc index fcb6d8e1..4ebd07c2 100644 --- a/credentials/credentials.cc +++ b/credentials/credentials.cc @@ -5,6 +5,7 @@ */ #include "credentials.h" +#include "soa/types/basic_value_descriptions.h" using namespace std; @@ -15,6 +16,20 @@ DEFINE_STRUCTURE_DESCRIPTION(Credential); CredentialDescription:: CredentialDescription() { + addField("provider", &Credential::provider, + "Provider of credentials"); + addField("protocol", &Credential::protocol, + "Protocol to use to access the service"); + addField("location", &Credential::location, + "Location of the service"); + addField("id", &Credential::id, + "User ID to use to access the service"); + addField("secret", &Credential::secret, + "Secret key to use to access the service"); + addField("extra", &Credential::extra, + "Extra configuration needed to access the service"); + addField("validUntil", &Credential::validUntil, + "Time until which the credential is valid"); } DEFINE_STRUCTURE_DESCRIPTION(CredentialContext); @@ -24,14 +39,4 @@ CredentialContextDescription() { } -std::vector -getCredentials(const std::string & resourceType, - const std::string & resource, - const CredentialContext & context, - Json::Value extraData) -{ - return {}; -} - - } // namespace Datacratic diff --git a/credentials/credentials.h b/credentials/credentials.h index 893fc2f5..2fcfaad2 100644 --- a/credentials/credentials.h +++ b/credentials/credentials.h @@ -7,21 +7,26 @@ #pragma once #include "soa/types/value_description.h" +#include "soa/types/date.h" +#include "soa/types/periodic_utils.h" namespace Datacratic { struct Credential { - std::string id; - std::string secret; + std::string provider; ///< Path through which credential was obtained + std::string protocol; ///< Protocol to use to get to service + std::string location; ///< URI to call to get resource + std::string id; ///< User ID + std::string secret; ///< Password / secret / etc - Json::Value metadata; + Json::Value extra; ///< Other fields + + Date validUntil; }; DECLARE_STRUCTURE_DESCRIPTION(Credential); struct CredentialContext { - std::string user; - Json::Value userid; }; DECLARE_STRUCTURE_DESCRIPTION(CredentialContext); @@ -34,6 +39,12 @@ std::vector getCredentials(const std::string & resourceType, const std::string & resource, const CredentialContext & context, - Json::Value extraData); + Json::Value extraData = Json::Value()); + +Credential getCredential(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context = CredentialContext(), + Json::Value extraData = Json::Value(), + TimePeriod validTime = "99999d"); } // namespace Datacratic diff --git a/service/aws.cc b/service/aws.cc index f1f72684..5daf2792 100644 --- a/service/aws.cc +++ b/service/aws.cc @@ -23,14 +23,14 @@ using namespace std; using namespace ML; - +#if 0 namespace { std::mutex awsCredentialsLock; std::map awsCredentials; } // file scope - +#endif namespace Datacratic { @@ -595,6 +595,7 @@ performGet(RestParams && params, resultSelector); } +#if 0 void registerAwsCredentials(const string & accessKeyId, const string & accessKey) { @@ -622,5 +623,6 @@ string getAwsAccessKey(const string & accessKeyId) return it->second; } +#endif } // namespace Datacratic diff --git a/service/aws.h b/service/aws.h index f29a8c3e..366494c3 100644 --- a/service/aws.h +++ b/service/aws.h @@ -174,6 +174,7 @@ struct AwsBasicApi : public AwsApi { HttpRestProxy proxy; }; +#if 0 /** Register an AWS access key for future referencing in urls or association * with buckets */ void registerAwsCredentials(const std::string & accessKeyId, @@ -181,5 +182,6 @@ void registerAwsCredentials(const std::string & accessKeyId, /** Returns the key associated with the access key id */ std::string getAwsAccessKey(const std::string & accessKeyId); +#endif } // namespace Datacratic diff --git a/service/s3.cc b/service/s3.cc index 1f6b3107..47cdf573 100644 --- a/service/s3.cc +++ b/service/s3.cc @@ -21,6 +21,8 @@ #include "jml/utils/file_functions.h" #include "jml/utils/info.h" #include "xml_helpers.h" +#include "soa/credentials/credentials.h" +#include "soa/credentials/credential_provider.h" #define CRYPTOPP_ENABLE_NAMESPACE_WEAK 1 #include "crypto++/sha.h" @@ -56,14 +58,14 @@ struct S3UrlFsHandler : public UrlFsHandler { virtual FsObjectInfo getInfo(const Url & url) const { string bucket = url.host(); - auto api = getS3ApiForBucket(bucket); + auto api = getS3ApiForUri(url.toString()); return api->getObjectInfo(bucket, url.path().substr(1)); } virtual FsObjectInfo tryGetInfo(const Url & url) const { string bucket = url.host(); - auto api = getS3ApiForBucket(bucket); + auto api = getS3ApiForUri(url.toString()); return api->tryGetObjectInfo(bucket, url.path().substr(1)); } @@ -74,7 +76,7 @@ struct S3UrlFsHandler : public UrlFsHandler { virtual bool erase(const Url & url, bool throwException) const { string bucket = url.host(); - auto api = getS3ApiForBucket(bucket); + auto api = getS3ApiForUri(url.toString()); if (throwException) { api->eraseObject(bucket, url.path()); return true; @@ -91,7 +93,7 @@ struct S3UrlFsHandler : public UrlFsHandler { const std::string & startAt) const { string bucket = prefix.host(); - auto api = getS3ApiForBucket(bucket); + auto api = getS3ApiForUri(prefix.toString()); bool result = true; @@ -206,6 +208,7 @@ init(const std::string & accessKeyId, this->bandwidthToServiceMbps = bandwidthToServiceMbps; } +#if 0 void S3Api::init() { @@ -222,6 +225,7 @@ S3Api::init() this->init(keyId, key); } +#endif S3Api::Content:: Content(const tinyxml2::XMLDocument & xml) @@ -452,10 +456,13 @@ performSync() const ::fprintf(stderr, "%s\n", message.c_str()); /* retry on 50X range errors (recoverable) */ - if (responseCode >= 500 and responseCode < 505) { + if (responseCode >= 500 && responseCode < 505) { continue; } else { + cerr << "Unrecoverable S3 error: code " << responseCode + << endl; + cerr << string(body, 0, 4096) << endl; throw ML::Exception("S3 error is unrecoverable"); } } @@ -2278,6 +2285,7 @@ parseUri(const std::string & uri) return make_pair(bucket, object); } +#if 0 void S3Handle:: initS3(const std::string & accessKeyId, @@ -2295,7 +2303,7 @@ getS3Buffer(const std::string & filename, char** outBuffer){ if (this->s3UriPrefix == "") { // not initialized; use defaults string bucket = S3Api::parseUri(filename).first; - auto api = getS3ApiForBucket(bucket); + auto api = getS3ApiForUri(filename); size_t size = api->getObjectInfo(filename).size; // cerr << "size = " << size << endl; @@ -2353,6 +2361,8 @@ getS3Buffer(const std::string & filename, char** outBuffer){ } +#endif + bool S3Api:: forEachBucket(const OnBucket & onBucket) const @@ -2432,6 +2442,7 @@ getDefaultRedundancy() return defaultRedundancy; } +#if 0 namespace { struct S3BucketInfo { @@ -2444,52 +2455,77 @@ std::unordered_map s3Buckets; } // file scope -/** S3 support for filter_ostream opens. Register the bucket name here, and - you can open it directly from s3. -*/ +#endif -void registerS3Bucket(const std::string & bucketName, - const std::string & accessKeyId, - const std::string & accessKey, - double bandwidthToServiceMbps, - const std::string & protocol, - const std::string & serviceUri) +std::string getEnv(const char * varName) { - std::unique_lock guard(s3BucketsLock); + const char * val = getenv(varName); + return val ? val : ""; +} - auto it = s3Buckets.find(bucketName); - if(it != s3Buckets.end()){ - shared_ptr api = it->second.api; - //if the info is different, raise an exception, otherwise return - if (api->accessKeyId != accessKeyId - || api->accessKey != accessKey - || api->bandwidthToServiceMbps != bandwidthToServiceMbps - || api->defaultProtocol != protocol - || api->serviceUri != serviceUri) - { - return; - throw ML::Exception("Trying to re-register a bucket with different " - "parameters"); - } - return; + +struct S3ExplicitCredentialProvider: public CredentialProvider { + + std::vector buckets; + Credential cred; + + S3ExplicitCredentialProvider() + { } - S3BucketInfo info; - info.s3Bucket = bucketName; - info.api = std::make_shared(accessKeyId, accessKey, - bandwidthToServiceMbps, - protocol, serviceUri); - info.api->getEscaped("", "/" + bucketName + "/", - S3Api::Range::Full); //throws if !accessible - s3Buckets[bucketName] = info; + S3ExplicitCredentialProvider(std::string provider, + std::string id, + std::string secret, + std::vector buckets, + double bandwidthToServiceMbps, + const std::string & protocol, + const std::string & serviceUri) + { + init(provider, id, secret, buckets, bandwidthToServiceMbps, protocol, serviceUri); + } - if (accessKeyId.size() > 0 && accessKey.size() > 0) { - registerAwsCredentials(accessKeyId, accessKey); + void init(std::string provider, + std::string id, + std::string secret, + std::vector buckets, + double bandwidthToServiceMbps = S3Api::defaultBandwidthToServiceMbps, + const std::string & protocol = "http", + const std::string & serviceUri = "s3.amazonaws.com") + { + cred.provider = provider; + cred.protocol = protocol; + cred.location = serviceUri; + cred.id = id; + cred.secret = secret; + cred.extra["bandwithToServiceMbps"] = bandwidthToServiceMbps; + this->buckets = buckets; } -} -/** Register S3 with the filter streams API so that a filter_stream can be used to - treat an S3 object as a simple stream. + + virtual std::vector + getResourceTypePrefixes() const + { + return { "aws:s3" }; + } + + virtual std::vector + getSync(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context, + Json::Value extraData) const + { + string bucket = S3Api::parseUri(resource).first; + if (!buckets.empty() + && (std::find(buckets.begin(), buckets.end(), bucket) + == buckets.end())) + return {}; + + return { cred }; + } +}; + +/** Register S3 with the filter streams API so that a filter_stream can be + used to treat an S3 object as a simple stream. */ struct RegisterS3Handler { static std::pair @@ -2559,104 +2595,202 @@ struct RegisterS3Handler { else throw ML::Exception("no way to create s3 handler for non in/out"); } - void registerBuckets() - { - } + /** Parse the ~/.cloud_credentials file and add those buckets in. + + The format of that file is as follows: + 1. One entry per line + 2. Tab separated + 3. Comments are '#' in the first position + 4. First entry is the name of the URI scheme (here, s3) + 5. Second entry is the "version" of the configuration (here, 1) + for forward compatibility + 6. The rest of the entries depend upon the scheme; for s3 they are + tab-separated and include the following: + - Access key ID + - Access key + - Bandwidth from this machine to the server (MBPS) + - Protocol (http) + - S3 machine host name (s3.amazonaws.com) + */ + struct CloudCredentialProvider: public CredentialProvider { + + CloudCredentialProvider() + { + // Parse cloud credentials file + + string filename = ""; + const char * home = getenv("HOME"); + if (home != NULL) + filename = home + string("/.cloud_credentials"); + if (filename != "" && ML::fileExists(filename)) { + std::ifstream stream(filename.c_str()); + int lineNum = 1; + for (; stream; ++lineNum) { + string line; + + getline(stream, line); + if (line.empty() || line[0] == '#') + continue; + if (line.find("s3") != 0) + continue; + + vector fields = ML::split(line, '\t'); + + if (fields[0] != "s3") + continue; + + if (fields.size() < 4) { + cerr << "warning: skipping invalid line in ~/.cloud_credentials: " + << line << endl; + continue; + } + + fields.resize(7); + + string version = fields[1]; + if (version != "1") { + cerr << "warning: ignoring unknown version " + << version << " in ~/.cloud_credentials: " + << line << endl; + continue; + } + + string keyId = fields[2]; + string key = fields[3]; + string bandwidth = fields[4]; + string protocol = fields[5]; + string serviceUri = fields[6]; + + double bw = S3Api::defaultBandwidthToServiceMbps; + if (bandwidth != "") + bw = boost::lexical_cast(bandwidth); + if (protocol == "") + protocol = "http"; + if (serviceUri == "") + serviceUri = "s3.amazonaws.com"; + + Credential cred; + cred.provider = "S3CloudCredentials " + filename + ":" + std::to_string(lineNum); + cred.id = keyId; + cred.secret = key; + cred.protocol = protocol; + cred.location = serviceUri; + cred.extra["bandwidthToServiceMbps"] = bw; + + creds.push_back(cred); + } + } + } + + std::vector creds; + + virtual std::vector + getResourceTypePrefixes() const + { + return { "aws:s3" }; + } + + virtual std::vector + getSync(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context, + Json::Value extraData) const + { + return creds; + } + }; + + struct S3EnvironmentCredentialProvider: public S3ExplicitCredentialProvider { + + std::vector buckets; + Credential cred; + + S3EnvironmentCredentialProvider() + { + init("S3EnvironmentCredentialProvider", + getEnv("S3_KEY_ID"), + getEnv("S3_KEY"), + ML::split(getEnv("S3_BUCKETS"), ',')); + } + }; RegisterS3Handler() { ML::registerUriHandler("s3", getS3Handler); + CredentialProvider::registerProvider + ("s3CloudCredentials", + std::make_shared()); + + if (getenv("S3_KEY_ID")) + CredentialProvider::registerProvider + ("s3FromEnvironment", + std::make_shared()); } } registerS3Handler; +#if 0 bool defaultBucketsRegistered = false; std::mutex registerBucketsMutex; +#endif -tuple getCloudCredentials() -{ - string filename = ""; - char* home; - home = getenv("HOME"); - if (home != NULL) - filename = home + string("/.cloud_credentials"); - if (filename != "" && ML::fileExists(filename)) { - std::ifstream stream(filename.c_str()); - while (stream) { - string line; - - getline(stream, line); - if (line.empty() || line[0] == '#') - continue; - if (line.find("s3") != 0) - continue; - - vector fields = ML::split(line, '\t'); - if (fields[0] != "s3") - continue; +/** S3 support for filter_ostream opens. Register the bucket name here, and + you can open it directly from s3. +*/ - if (fields.size() < 4) { - cerr << "warning: skipping invalid line in ~/.cloud_credentials: " - << line << endl; - continue; - } - - fields.resize(7); +void registerS3Bucket(const std::string & bucketName, + const std::string & accessKeyId, + const std::string & accessKey, + double bandwidthToServiceMbps, + const std::string & protocol, + const std::string & serviceUri) +{ + CredentialProvider::registerProvider + ("s3UserRegisteredBucket", + std::make_shared + ("registerS3Bucket()", accessKeyId, accessKey, + vector({ bucketName }), + bandwidthToServiceMbps, protocol, serviceUri)); - string version = fields[1]; - if (version != "1") { - cerr << "warning: ignoring unknown version " - << version << " in ~/.cloud_credentials: " - << line << endl; - continue; - } - - string keyId = fields[2]; - string key = fields[3]; - string bandwidth = fields[4]; - string protocol = fields[5]; - string serviceUri = fields[6]; +#if 0 + std::unique_lock guard(s3BucketsLock); - return make_tuple(keyId, key, bandwidth, protocol, serviceUri); + auto it = s3Buckets.find(bucketName); + if(it != s3Buckets.end()){ + shared_ptr api = it->second.api; + //if the info is different, raise an exception, otherwise return + if (api->accessKeyId != accessKeyId + || api->accessKey != accessKey + || api->bandwidthToServiceMbps != bandwidthToServiceMbps + || api->defaultProtocol != protocol + || api->serviceUri != serviceUri) + { + return; + throw ML::Exception("Trying to re-register a bucket with different " + "parameters"); } + return; } - return make_tuple("", "", "", "", ""); -} -std::string getEnv(const char * varName) -{ - const char * val = getenv(varName); - return val ? val : ""; + S3BucketInfo info; + info.s3Bucket = bucketName; + info.api = std::make_shared(accessKeyId, accessKey, + bandwidthToServiceMbps, + protocol, serviceUri); + info.api->getEscaped("", "/" + bucketName + "/", + S3Api::Range::Full); //throws if !accessible + s3Buckets[bucketName] = info; +#endif + +#if 0 + if (accessKeyId.size() > 0 && accessKey.size() > 0) { + registerAwsCredentials(accessKeyId, accessKey); + } +#endif } -tuple > -getS3CredentialsFromEnvVar() -{ - return make_tuple(getEnv("S3_KEY_ID"), getEnv("S3_KEY"), - ML::split(getEnv("S3_BUCKETS"), ',')); -} - -/** Parse the ~/.cloud_credentials file and add those buckets in. - - The format of that file is as follows: - 1. One entry per line - 2. Tab separated - 3. Comments are '#' in the first position - 4. First entry is the name of the URI scheme (here, s3) - 5. Second entry is the "version" of the configuration (here, 1) - for forward compatibility - 6. The rest of the entries depend upon the scheme; for s3 they are - tab-separated and include the following: - - Access key ID - - Access key - - Bandwidth from this machine to the server (MBPS) - - Protocol (http) - - S3 machine host name (s3.amazonaws.com) - - If S3_KEY_ID and S3_KEY environment variables are specified, - they will be used first. -*/ +#if 0 void registerDefaultBuckets() { if (defaultBucketsRegistered) @@ -2731,40 +2865,45 @@ void registerDefaultBuckets() #endif } +#endif + void registerS3Buckets(const std::string & accessKeyId, const std::string & accessKey, double bandwidthToServiceMbps, const std::string & protocol, const std::string & serviceUri) { - std::unique_lock guard(s3BucketsLock); - int bucketCount(0); auto api = std::make_shared(accessKeyId, accessKey, bandwidthToServiceMbps, protocol, serviceUri); + vector bucketNames; + auto onBucket = [&] (const std::string & bucketName) { - //cerr << "got bucket " << bucketName << endl; - - S3BucketInfo info; - info.s3Bucket = bucketName; - info.api = api; - s3Buckets[bucketName] = info; - bucketCount++; - + bucketNames.push_back(bucketName); return true; }; api->forEachBucket(onBucket); - if (bucketCount == 0) { - cerr << "registerS3Buckets: no bucket registered\n"; + if (bucketNames.empty()) { + cerr << "warning: no bucket names registered"; + } else { + CredentialProvider::registerProvider + ("s3UserRegisteredBucket", + std::make_shared + ("registerS3Buckets()", accessKeyId, accessKey, + bucketNames, bandwidthToServiceMbps, protocol, serviceUri)); } } +#if 0 std::shared_ptr getS3ApiForBucket(const std::string & bucketName) { + auto creds = getCredential("aws:s3", uri); + return std::make_shared(creds.id, creds.secret); + std::unique_lock guard(s3BucketsLock); auto it = s3Buckets.find(bucketName); if (it == s3Buckets.end()) { @@ -2777,8 +2916,24 @@ std::shared_ptr getS3ApiForBucket(const std::string & bucketName) } return it->second.api; } +#endif + +double getBandwidth(const Credential & cred) +{ + if (cred.extra.isMember("bandwidthToServiceMbps")) + return cred.extra["bandwidthToServiceMbps"].asDouble(); + else return S3Api::defaultBandwidthToServiceMbps; +} std::shared_ptr getS3ApiForUri(const std::string & uri) +{ + // Get the credentials + auto creds = getCredential("aws:s3", uri); + return std::make_shared(creds.id, creds.secret, getBandwidth(creds), + creds.protocol, creds.location); +} + +#if 0 { Url url(uri); @@ -2794,11 +2949,11 @@ std::shared_ptr getS3ApiForUri(const std::string & uri) } auto api = make_shared(accessKeyId, accessKey); - api->getEscaped("", "/" + bucketName + "/", - S3Api::Range::Full); //throws if !accessible + //api->getEscaped("", "/" + bucketName + "/", + // S3Api::Range::Full); //throws if !accessible return api; } - +#endif } // namespace Datacratic diff --git a/service/s3.h b/service/s3.h index 03f901b3..2f83a718 100644 --- a/service/s3.h +++ b/service/s3.h @@ -77,8 +77,12 @@ struct S3Api : public AwsApi { const std::string & defaultProtocol = "http", const std::string & serviceUri = "s3.amazonaws.com"); + /** Set up the API, getting its credentials from the default credentials + handler. + */ + //void init(); + /** Set up the API to called with the given credentials. */ - void init(); void init(const std::string & accessKeyId, const std::string & accessKey, double bandwidthToServiceMbps = defaultBandwidthToServiceMbps, @@ -632,17 +636,6 @@ struct S3Api : public AwsApi { }; -struct S3Handle{ - S3Api s3; - std::string s3UriPrefix; - - void initS3(const std::string & accessKeyId, - const std::string & accessKey, - const std::string & uriPrefix); - - size_t getS3Buffer(const std::string & filename, char** outBuffer); -}; - /** S3 support for filter_ostream opens. Register the bucket name here, and you can open it directly from s3. */ @@ -665,10 +658,13 @@ void registerS3Buckets(const std::string & accessKeyId, const std::string & protocol = "http", const std::string & serviceUri = "s3.amazonaws.com"); +#if 0 std::shared_ptr getS3ApiForBucket(const std::string & bucketName); +#endif std::shared_ptr getS3ApiForUri(const std::string & uri); +#if 0 std::tuple getCloudCredentials(); @@ -680,6 +676,8 @@ std::tuple std::tuple > getS3CredentialsFromEnvVar(); +#endif + // std::pair getDefaultCredentials(); } // namespace Datacratic From 1796331f2b6c47e1b8b200a20f25a5cf409084d8 Mon Sep 17 00:00:00 2001 From: Jeremy Barnes Date: Thu, 6 Nov 2014 16:25:23 -0500 Subject: [PATCH 09/15] Cleaned up dead code from S3 refactor --- service/aws.cc | 39 ------- service/aws.h | 10 -- service/s3.cc | 285 ++----------------------------------------------- service/s3.h | 26 ++--- 4 files changed, 15 insertions(+), 345 deletions(-) diff --git a/service/aws.cc b/service/aws.cc index 5daf2792..8fa8dd70 100644 --- a/service/aws.cc +++ b/service/aws.cc @@ -23,15 +23,6 @@ using namespace std; using namespace ML; -#if 0 -namespace { - -std::mutex awsCredentialsLock; -std::map awsCredentials; - -} // file scope -#endif - namespace Datacratic { template @@ -595,34 +586,4 @@ performGet(RestParams && params, resultSelector); } -#if 0 -void registerAwsCredentials(const string & accessKeyId, - const string & accessKey) -{ - unique_lock guard(awsCredentialsLock); - - string & entry = awsCredentials[accessKeyId]; - if (entry.empty()) { - entry = accessKey; - } - else { - if (entry != accessKey) { - throw ML::Exception("access key id '%s' already registered with a" - " different key", accessKeyId.c_str()); - } - } -} - -string getAwsAccessKey(const string & accessKeyId) -{ - auto it = awsCredentials.find(accessKeyId); - if (it == awsCredentials.end()) { - throw ML::Exception("no access key registered for id '%s'", - accessKeyId.c_str()); - } - - return it->second; -} -#endif - } // namespace Datacratic diff --git a/service/aws.h b/service/aws.h index 366494c3..ef883ea7 100644 --- a/service/aws.h +++ b/service/aws.h @@ -174,14 +174,4 @@ struct AwsBasicApi : public AwsApi { HttpRestProxy proxy; }; -#if 0 -/** Register an AWS access key for future referencing in urls or association - * with buckets */ -void registerAwsCredentials(const std::string & accessKeyId, - const std::string & accessKey); - -/** Returns the key associated with the access key id */ -std::string getAwsAccessKey(const std::string & accessKeyId); -#endif - } // namespace Datacratic diff --git a/service/s3.cc b/service/s3.cc index 47cdf573..1e8f3601 100644 --- a/service/s3.cc +++ b/service/s3.cc @@ -208,25 +208,6 @@ init(const std::string & accessKeyId, this->bandwidthToServiceMbps = bandwidthToServiceMbps; } -#if 0 -void -S3Api::init() -{ - string keyId, key; - std::tie(keyId, key, std::ignore) - = getS3CredentialsFromEnvVar(); - - if (keyId == "" || key == "") { - tie(keyId, key, std::ignore, std::ignore, std::ignore) - = getCloudCredentials(); - } - if (keyId == "" || key == "") - throw ML::Exception("Cannot init S3 API with no keys, environment or creedentials file"); - - this->init(keyId, key); -} -#endif - S3Api::Content:: Content(const tinyxml2::XMLDocument & xml) { @@ -2285,84 +2266,6 @@ parseUri(const std::string & uri) return make_pair(bucket, object); } -#if 0 -void -S3Handle:: -initS3(const std::string & accessKeyId, - const std::string & accessKey, - const std::string & uriPrefix) -{ - s3.init(accessKeyId, accessKey); - this->s3UriPrefix = uriPrefix; -} - -size_t -S3Handle:: -getS3Buffer(const std::string & filename, char** outBuffer){ - - if (this->s3UriPrefix == "") { - // not initialized; use defaults - string bucket = S3Api::parseUri(filename).first; - auto api = getS3ApiForUri(filename); - size_t size = api->getObjectInfo(filename).size; - - // cerr << "size = " << size << endl; - - // TODO: outBuffer exception safety - *outBuffer = new char[size]; - - uint64_t done = 0; - - auto onChunk = [&] (const char * data, - size_t chunkSize, - int chunkIndex, - uint64_t offset, - uint64_t totalSize) - { - ExcAssertEqual(size, totalSize); - ExcAssertLessEqual(offset + chunkSize, totalSize); - std::copy(data, data + chunkSize, *outBuffer + offset); - ML::atomic_add(done, chunkSize); - }; - - api->download(filename, onChunk); - - return size; - } - - auto stats = s3.getObjectInfo(filename); - if (!stats) - throw ML::Exception("unknown s3 object"); - - *outBuffer = new char[stats.size]; - - uint64_t done = 0; - - auto onChunk = [&] (const char * data, - size_t size, - int chunkIndex, - uint64_t offset, - uint64_t totalSize) - { - ExcAssertEqual(stats.size, totalSize); - ExcAssertLessEqual(offset + size, totalSize); - std::copy(data, data + size, *outBuffer + offset); - ML::atomic_add(done, size); - }; - - s3.download(filename, onChunk); - - ExcAssertEqual(done, stats.size); - - // cerr << "done downloading " << stats.size << " bytes from " - // << filename << endl; - - return stats.size; - -} - -#endif - bool S3Api:: forEachBucket(const OnBucket & onBucket) const @@ -2442,27 +2345,18 @@ getDefaultRedundancy() return defaultRedundancy; } -#if 0 -namespace { - -struct S3BucketInfo { - std::string s3Bucket; - std::shared_ptr api; //< Used to access this uri -}; - -std::recursive_mutex s3BucketsLock; -std::unordered_map s3Buckets; - -} // file scope - -#endif - -std::string getEnv(const char * varName) +/** getEnv, but compatible with std::string. Returns null string if not + found. +*/ +static std::string getEnv(const char * varName) { const char * val = getenv(varName); return val ? val : ""; } +/** Provider of S3 credentials that are added explicitly for a subset of + buckets. +*/ struct S3ExplicitCredentialProvider: public CredentialProvider { @@ -2729,12 +2623,6 @@ struct RegisterS3Handler { } registerS3Handler; -#if 0 -bool defaultBucketsRegistered = false; -std::mutex registerBucketsMutex; -#endif - - /** S3 support for filter_ostream opens. Register the bucket name here, and you can open it directly from s3. */ @@ -2752,121 +2640,8 @@ void registerS3Bucket(const std::string & bucketName, ("registerS3Bucket()", accessKeyId, accessKey, vector({ bucketName }), bandwidthToServiceMbps, protocol, serviceUri)); - -#if 0 - std::unique_lock guard(s3BucketsLock); - - auto it = s3Buckets.find(bucketName); - if(it != s3Buckets.end()){ - shared_ptr api = it->second.api; - //if the info is different, raise an exception, otherwise return - if (api->accessKeyId != accessKeyId - || api->accessKey != accessKey - || api->bandwidthToServiceMbps != bandwidthToServiceMbps - || api->defaultProtocol != protocol - || api->serviceUri != serviceUri) - { - return; - throw ML::Exception("Trying to re-register a bucket with different " - "parameters"); - } - return; - } - - S3BucketInfo info; - info.s3Bucket = bucketName; - info.api = std::make_shared(accessKeyId, accessKey, - bandwidthToServiceMbps, - protocol, serviceUri); - info.api->getEscaped("", "/" + bucketName + "/", - S3Api::Range::Full); //throws if !accessible - s3Buckets[bucketName] = info; -#endif - -#if 0 - if (accessKeyId.size() > 0 && accessKey.size() > 0) { - registerAwsCredentials(accessKeyId, accessKey); - } -#endif } -#if 0 -void registerDefaultBuckets() -{ - if (defaultBucketsRegistered) - return; - - std::unique_lock guard(registerBucketsMutex); - defaultBucketsRegistered = true; - - tuple cloudCredentials = - getCloudCredentials(); - if (get<0>(cloudCredentials) != "") { - string keyId = get<0>(cloudCredentials); - string key = get<1>(cloudCredentials); - string bandwidth = get<2>(cloudCredentials); - string protocol = get<3>(cloudCredentials); - string serviceUri = get<4>(cloudCredentials); - - if (protocol == "") - protocol = "http"; - if (bandwidth == "") - bandwidth = "20.0"; - if (serviceUri == "") - serviceUri = "s3.amazonaws.com"; - - registerS3Buckets(keyId, key, boost::lexical_cast(bandwidth), - protocol, serviceUri); - return; - } - string keyId; - string key; - vector buckets; - - std::tie(keyId, key, buckets) = getS3CredentialsFromEnvVar(); - if (keyId != "" && key != "") { - if (buckets.empty()) { - registerS3Buckets(keyId, key); - } - else { - for (string bucket: buckets) - registerS3Bucket(bucket, keyId, key); - } - } - else - cerr << "WARNING: registerDefaultBuckets needs either a " - ".cloud_credentials or S3_KEY_ID and S3_KEY environment " - " variables" << endl; - -#if 0 - char* configFilenameCStr = getenv("CONFIG"); - string configFilename = (configFilenameCStr == NULL ? - string() : - string(configFilenameCStr)); - - if(configFilename != "") - { - ML::File_Read_Buffer buf(configFilename); - Json::Value config = Json::parse(string(buf.start(), buf.end())); - if(config.isMember("s3")) - { - registerS3Buckets( - config["s3"]["accessKeyId"].asString(), - config["s3"]["accessKey"].asString(), - 20., - "http", - "s3.amazonaws.com"); - return; - } - } - cerr << "WARNING: registerDefaultBuckets needs either a .cloud_credentials" - " file or an environment variable CONFIG pointing toward a file " - "having keys s3.accessKey and s3.accessKeyId" << endl; -#endif -} - -#endif - void registerS3Buckets(const std::string & accessKeyId, const std::string & accessKey, double bandwidthToServiceMbps, @@ -2898,27 +2673,8 @@ void registerS3Buckets(const std::string & accessKeyId, } } -#if 0 -std::shared_ptr getS3ApiForBucket(const std::string & bucketName) -{ - auto creds = getCredential("aws:s3", uri); - return std::make_shared(creds.id, creds.secret); - - std::unique_lock guard(s3BucketsLock); - auto it = s3Buckets.find(bucketName); - if (it == s3Buckets.end()) { - // On demand, load up the configuration file before we fail - registerDefaultBuckets(); - it = s3Buckets.find(bucketName); - } - if (it == s3Buckets.end()) { - throw ML::Exception("unregistered s3 bucket " + bucketName); - } - return it->second.api; -} -#endif - -double getBandwidth(const Credential & cred) +/** Extract the bandwidth to the service from a credentials object. */ +static double getBandwidth(const Credential & cred) { if (cred.extra.isMember("bandwidthToServiceMbps")) return cred.extra["bandwidthToServiceMbps"].asDouble(); @@ -2933,27 +2689,4 @@ std::shared_ptr getS3ApiForUri(const std::string & uri) creds.protocol, creds.location); } -#if 0 -{ - Url url(uri); - - string bucketName = url.host(); - string accessKeyId = url.username(); - if (accessKeyId.empty()) { - return getS3ApiForBucket(bucketName); - } - - string accessKey = url.password(); - if (accessKey.empty()) { - accessKey = getAwsAccessKey(accessKeyId); - } - - auto api = make_shared(accessKeyId, accessKey); - //api->getEscaped("", "/" + bucketName + "/", - // S3Api::Range::Full); //throws if !accessible - - return api; -} -#endif - } // namespace Datacratic diff --git a/service/s3.h b/service/s3.h index 2f83a718..60cfb117 100644 --- a/service/s3.h +++ b/service/s3.h @@ -658,26 +658,12 @@ void registerS3Buckets(const std::string & accessKeyId, const std::string & protocol = "http", const std::string & serviceUri = "s3.amazonaws.com"); -#if 0 -std::shared_ptr getS3ApiForBucket(const std::string & bucketName); -#endif - -std::shared_ptr getS3ApiForUri(const std::string & uri); - -#if 0 -std::tuple - getCloudCredentials(); - -/** Returns the keyId, key and list of buckets to register (can be empty, - which means all) from the environment variables - - S3_KEY_ID, S3_KEY and S3_BUCKETS +/** Returns an S3Api constructed to access the given URI. Will look up its + own credentials using registered credential providers, or one which was + registered using registerS3Bucket or registerS3Buckets, or + ~/.cloud_credentials, or S3_KEY_ID, S3_KEY and S3_BUCKETS environment + variables. */ -std::tuple > -getS3CredentialsFromEnvVar(); - -#endif - -// std::pair getDefaultCredentials(); +std::shared_ptr getS3ApiForUri(const std::string & uri); } // namespace Datacratic From 19f9ef3ccef026dbefa99b1cf0f572b32396b80f Mon Sep 17 00:00:00 2001 From: Jeremy Barnes Date: Thu, 13 Nov 2014 10:06:55 -0500 Subject: [PATCH 10/15] Allow cloud credentials to be ignored programatically --- service/s3.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/service/s3.cc b/service/s3.cc index 1e8f3601..c1a77209 100644 --- a/service/s3.cc +++ b/service/s3.cc @@ -2418,6 +2418,8 @@ struct S3ExplicitCredentialProvider: public CredentialProvider { } }; +bool disableCloudCredentials = false; + /** Register S3 with the filter streams API so that a filter_stream can be used to treat an S3 object as a simple stream. */ @@ -2590,6 +2592,8 @@ struct RegisterS3Handler { const CredentialContext & context, Json::Value extraData) const { + if (disableCloudCredentials) + return {}; return creds; } }; From 12b2aa53b4ecb87e040e2fae2c2d32cbcd4bf200 Mon Sep 17 00:00:00 2001 From: Jeremy Barnes Date: Thu, 13 Nov 2014 10:07:39 -0500 Subject: [PATCH 11/15] Added ability to get credentials from a remote provider --- credentials/credential_provider.cc | 23 ++++ service/remote_credential_provider.cc | 168 ++++++++++++++++++++++++++ service/remote_credential_provider.h | 91 ++++++++++++++ service/service.mk | 3 +- 4 files changed, 284 insertions(+), 1 deletion(-) create mode 100644 service/remote_credential_provider.cc create mode 100644 service/remote_credential_provider.h diff --git a/credentials/credential_provider.cc b/credentials/credential_provider.cc index 75ba679a..6f1eb5ac 100644 --- a/credentials/credential_provider.cc +++ b/credentials/credential_provider.cc @@ -72,10 +72,33 @@ getCredential(const std::string & resourceType, { std::unique_lock guard(providersLock); + cerr << "getCredential" << endl; + + for (auto it = providers.begin(), end = providers.end(); + it != end; ++it) { + cerr << "testing " << it->first << " against " << resourceType + << " " << resource << endl; + if (resourceType.find(it->first) != 0) + break; // not a prefix + cerr << "FOUND" << endl; + + auto creds = it->second->getSync(resourceType, resource, context, + extraData); + if (!creds.empty()) { + cerr << "credentials for " << resourceType << " " << resource + << " are " << endl << jsonEncode(creds[0]) << endl; + return creds[0]; + } + } + for (auto it = providers.lower_bound(resourceType); it != providers.end(); ++it) { + cerr << "testing " << it->first << " against " << resourceType + << endl; if (resourceType.find(it->first) != 0) break; // not a prefix + cerr << "FOUND" << endl; + auto creds = it->second->getSync(resourceType, resource, context, extraData); if (!creds.empty()) { diff --git a/service/remote_credential_provider.cc b/service/remote_credential_provider.cc new file mode 100644 index 00000000..48bb21cb --- /dev/null +++ b/service/remote_credential_provider.cc @@ -0,0 +1,168 @@ +/** remote_credential_provider.cc + Jeremy Barnes, 12 November 2014 + Copyright (c) 2014 Datacratic Inc. All rights reserved. + + Credentials provider that gets credentials from a remote source. +*/ + +#include "remote_credential_provider.h" + + +using namespace std; + + +namespace Datacratic { + +/*****************************************************************************/ +/* CREDENTIALS DAEMON CLIENT */ +/*****************************************************************************/ + +CredentialsDaemonClient:: +CredentialsDaemonClient(const std::string & uri) +{ + connect(uri); +} + +CredentialsDaemonClient:: +~CredentialsDaemonClient() +{ +} + +void +CredentialsDaemonClient:: +connect(const std::string & uri) +{ + if (uri.find("http") != 0) { + throw ML::Exception("'uri' parameter does not start with http: " + uri); + } + conn.init(uri); + + try { + auto resp = conn.get("/info"); + if (resp.code() != 200) { + throw ML::Exception("Failed to connect to the credentials daemon"); + } + } + catch (const std::exception & exc) { + cerr << "Failed to connect to the credentials daemon: " << exc.what() + << endl; + throw; + } +} + +std::vector +CredentialsDaemonClient:: +getCredentials(const std::string & resourceType, + const std::string & resource, + const std::string & role, + const std::string & operation, + const TimePeriod & validity, + const Json::Value & extra) const +{ + cerr << "getCredentials " << resourceType << " " << resource << endl; + + string uri = "/v1/types/" + resourceType + "/resources/" + resource + + "/credentials"; + RestParams params; + if (role != "") + params.emplace_back("role", role); + if (operation != "") + params.emplace_back("operation", operation); + if (validity != TimePeriod()) + params.emplace_back("validity", validity.toString()); + if (!extra.isNull()) + params.emplace_back("extra", extra.toStringNoNewLine()); + + cerr << "calling" << endl; + + auto res = conn.get(uri, params, {}, 5.0 /* seconds */); + + cerr << "res = " << res << endl; + + if (res.code() != 200) + throw ML::Exception("couldn't get credentials: returned code %d", + res.code()); + + return jsonDecodeStr >(res.body()); +} + + +/*****************************************************************************/ +/* REMOTE CREDENTIAL PROVIDER */ +/*****************************************************************************/ + +RemoteCredentialProvider:: +RemoteCredentialProvider(const std::string & uri) +{ + client.connect(uri); +} + +std::vector +RemoteCredentialProvider:: +getResourceTypePrefixes() const +{ + return {""}; // for all prefixes +} + +std::vector +RemoteCredentialProvider:: +getSync(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context, + Json::Value extraData) const +{ + cerr << "Remote getSync for " << resourceType << " " << resource + << endl; + + std::string role = "default"; + std::string operation = "*"; + TimePeriod validity = "10000d"; + + return client.getCredentials(resourceType, resource, role, operation, + validity, extraData); +} + + +/*****************************************************************************/ +/* REGISTRATION */ +/*****************************************************************************/ + +void addRemoteCredentialProvider(const std::string & uri) +{ + cerr << "Registering remote credential provider at " << uri << endl; + + auto provider = std::make_shared(uri); + + CredentialProvider::registerProvider("Remote provider at " + uri, + provider); +} + +void initRemoteCredentialsFromEnvironment() +{ + for (const char * const * p = environ; *p; ++p) { + string var(*p); + if (var.find("REMOTE_CREDENTIAL_PROVIDER") == 0) { + auto equalPos = var.find('='); + if (equalPos == string::npos) + continue; // strange, no equals + string val(var, equalPos + 1); + + addRemoteCredentialProvider(val); + } + } +} + +namespace { + +struct AtInit { + + AtInit() + { + initRemoteCredentialsFromEnvironment(); + } + +} atInit; + +} // file scope + +} // namespace Datacratic diff --git a/service/remote_credential_provider.h b/service/remote_credential_provider.h new file mode 100644 index 00000000..49b3cf0d --- /dev/null +++ b/service/remote_credential_provider.h @@ -0,0 +1,91 @@ +/** remote_credential_provider.cc + Jeremy Barnes, 12 November 2014 + Copyright (c) 2014 Datacratic Inc. All rights reserved. + + Credentials provider that gets credentials from a remote source. +*/ + + +#pragma once + +#include "soa/credentials/credentials.h" +#include "soa/credentials/credential_provider.h" +#include "soa/service/http_rest_proxy.h" +#include "soa/credentials/credentials.h" +#include "soa/types/periodic_utils.h" + +namespace Datacratic { + + +/*****************************************************************************/ +/* CREDENTIALS DAEMON CLIENT */ +/*****************************************************************************/ + +/** A client for a credentials daemon. + + By writing a daemon, it is possible to implement logic like short-lived + credentials. + + The daemon must respond to the following routes: + 1. GET /interfaces/credentialsd + 2. GET /v1/types//resources//credentials?role=...&operation=...&validity=...&extra=... + +*/ + +struct CredentialsDaemonClient { + CredentialsDaemonClient() = default; + CredentialsDaemonClient(const std::string & uri); + + ~CredentialsDaemonClient(); + + void connect(const std::string & uri); + + std::vector + getCredentials(const std::string & resourceType, + const std::string & resource, + const std::string & role, + const std::string & operation = "*", + const TimePeriod & validity = "1h", + const Json::Value & extra = Json::Value()) const; + +protected: + HttpRestProxy conn; +}; + + +/*****************************************************************************/ +/* REMOTE CREDENTIAL PROVIDER */ +/*****************************************************************************/ + +struct RemoteCredentialProvider: public CredentialProvider { + + RemoteCredentialProvider(const std::string & uri); + + CredentialsDaemonClient client; + + virtual std::vector + getResourceTypePrefixes() const; + + virtual std::vector + getSync(const std::string & resourceType, + const std::string & resource, + const CredentialContext & context, + Json::Value extraData) const; + +}; + +/*****************************************************************************/ +/* REGISTRATION */ +/*****************************************************************************/ + +/** Add the ability to get remote credentials from the given URI. */ +void addRemoteCredentialProvider(const std::string & uri); + +/** This function will scan the environment looking for any entries that + start with REMOTE_CREDENTIAL_PROVIDER, and for each of them will set + up the given URI as a place to look for remote credentials. +*/ +void initRemoteCredentialsFromEnvironment(); + +} // namespace Datacratic + diff --git a/service/service.mk b/service/service.mk index 152936c9..28f5a7e6 100644 --- a/service/service.mk +++ b/service/service.mk @@ -54,7 +54,8 @@ LIBSERVICES_SOURCES := \ http_rest_proxy.cc \ xml_helpers.cc \ nprobe.cc \ - logs.cc + logs.cc \ + remote_credential_provider.cc LIBSERVICES_LINK := opstats curl curlpp boost_regex zeromq zookeeper_mt ACE arch utils jsoncpp boost_thread zmq types tinyxml2 boost_system value_description credentials From 69ce3ad7a6dc3f9676becd70f34b256e88c3876f Mon Sep 17 00:00:00 2001 From: Jeremy Barnes Date: Thu, 13 Nov 2014 10:08:28 -0500 Subject: [PATCH 12/15] Fixed incorrect use of lexical_cast in RestParamDefault (TRIVIAL) --- service/rest_request_params.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/rest_request_params.h b/service/rest_request_params.h index d5265713..be5e2396 100644 --- a/service/rest_request_params.h +++ b/service/rest_request_params.h @@ -127,7 +127,7 @@ struct RestParamDefault { const std::string & description, T defaultValue = T()) : name(name), description(description), defaultValue(defaultValue), - defaultValueStr(boost::lexical_cast(defaultValue)) + defaultValueStr(Codec::encode(defaultValue)) { //std::cerr << "created RestParam with " << name << " at " // << this << std::endl; From 3d97b4633e3ec8fbcdfb4fb6af87c0bf44840fcb Mon Sep 17 00:00:00 2001 From: Jeremy Barnes Date: Thu, 13 Nov 2014 10:09:28 -0500 Subject: [PATCH 13/15] Added TRACE_REST_REQUESTS environment variable (TRIVIAL) --- service/rest_request_router.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/service/rest_request_router.cc b/service/rest_request_router.cc index 464a941a..e2a010da 100644 --- a/service/rest_request_router.cc +++ b/service/rest_request_router.cc @@ -8,6 +8,7 @@ #include "jml/utils/vector_utils.h" #include "jml/arch/exception_handler.h" #include "jml/utils/set_utils.h" +#include "jml/utils/environment.h" using namespace std; @@ -110,6 +111,12 @@ static std::string getVerbsStr(const std::set & verbs) return verbsStr; } +namespace { + +ML::Env_Option TRACE_REST_REQUESTS("TRACE_REST_REQUESTS", false); + +} // file scope + RestRequestRouter:: MatchResult RestRequestRouter:: @@ -117,7 +124,7 @@ processRequest(const RestServiceEndpoint::ConnectionId & connection, const RestRequest & request, RestRequestParsingContext & context) const { - bool debug = false; + bool debug = TRACE_REST_REQUESTS; if (debug) { cerr << "processing request " << request From 584c7c7b081fe54f5924fc5feb281f009e855fd4 Mon Sep 17 00:00:00 2001 From: Jeremy Barnes Date: Thu, 13 Nov 2014 10:10:21 -0500 Subject: [PATCH 14/15] Don't require service discovery to use NamedEndpoint (TRIVIAL) --- service/named_endpoint.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/service/named_endpoint.cc b/service/named_endpoint.cc index 8e4a310e..d35026e9 100644 --- a/service/named_endpoint.cc +++ b/service/named_endpoint.cc @@ -39,7 +39,9 @@ NamedEndpoint:: publishAddress(const std::string & address, const Json::Value & addressConfig) { - ExcAssert(config); + // If we didn't set up a configuration endpoint or name, then just no-op + if (!config) + return; //cerr << "publishing " << address << " with " << addressConfig << endl; config->setUnique(endpointName + "/" + address, From bb708e9fa44c5055c0e4d6a3f1bfbc257a435d72 Mon Sep 17 00:00:00 2001 From: Jeremy Barnes Date: Thu, 13 Nov 2014 15:34:31 -0500 Subject: [PATCH 15/15] Improvements to value descriptions for objects without --- types/value_description.h | 50 ++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/types/value_description.h b/types/value_description.h index b0a04030..72ed3c78 100644 --- a/types/value_description.h +++ b/types/value_description.h @@ -335,28 +335,6 @@ struct RegisterValueDescriptionI { static const RegisterValueDescription registerValueDescription##type; \ } -/*****************************************************************************/ -/* PURE VALUE DESCRIPTION */ -/*****************************************************************************/ - -template -struct PureValueDescription : public ValueDescription { - PureValueDescription() : - ValueDescription(ValueKind::ATOM, &typeid(T)) { - } - - virtual void parseJson(void * val, JsonParsingContext & context) const {}; - virtual void printJson(const void * val, JsonPrintingContext & context) const {}; - virtual bool isDefault(const void * val) const { return false; } - virtual void setDefault(void * val) const {} - virtual void copyValue(const void * from, void * to) const {} - virtual void moveValue(void * from, void * to) const {} - virtual void swapValues(void * from, void * to) const {} - virtual void * constructDefault() const {return nullptr;} - virtual void destroy(void *) const {} - -}; - /*****************************************************************************/ /* VALUE DESCRIPTION TEMPLATE */ /*****************************************************************************/ @@ -648,6 +626,28 @@ maybeGetDefaultDescriptionShared(T * = 0) return result; } +/*****************************************************************************/ +/* PURE VALUE DESCRIPTION */ +/*****************************************************************************/ + +template +struct PureValueDescription : public ValueDescriptionT { + PureValueDescription() : + ValueDescriptionT(ValueKind::ATOM) { + } + + virtual void parseJson(void * val, JsonParsingContext & context) const {}; + virtual void printJson(const void * val, JsonPrintingContext & context) const {}; + virtual bool isDefault(const void * val) const { return false; } + virtual void setDefault(void * val) const {} + virtual void copyValue(const void * from, void * to) const {} + virtual void moveValue(void * from, void * to) const {} + virtual void swapValues(void * from, void * to) const {} + virtual void * constructDefault() const {return nullptr;} + virtual void destroy(void *) const {} + +}; + /*****************************************************************************/ /* VALUE DESCRIPTION CONCRETE IMPL */ @@ -2028,3 +2028,9 @@ inline Json::Value jsonEncode(const char * str) #define CREATE_ENUM_DESCRIPTION(Type) \ CREATE_ENUM_DESCRIPTION_NAMED(Type##Description, Type) + +#define HAS_NO_VALUE_DESCRIPTION(Type) \ + inline PureValueDescription * getDefaultDescription(Type *) \ + { \ + return new PureValueDescription(); \ + }