diff --git a/.env b/.env index 42453ba..4c6059d 100644 --- a/.env +++ b/.env @@ -1,5 +1,5 @@ SERVICE_NAME=cds -OTP_VERSION=24.2.0 -REBAR_VERSION=3.18 +OTP_VERSION=27.1.2 +REBAR_VERSION=3.24 THRIFT_VERSION=0.14.2.2 RIAK_VERSION=ubuntu-2.2.3 diff --git a/.github/workflows/erlang-checks.yml b/.github/workflows/erlang-checks.yml index 9c8207a..3c2bc1a 100644 --- a/.github/workflows/erlang-checks.yml +++ b/.github/workflows/erlang-checks.yml @@ -18,7 +18,7 @@ jobs: thrift-version: ${{ steps.thrift-version.outputs.version }} steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - run: grep -v '^#' .env >> $GITHUB_ENV - id: otp-version run: echo "::set-output name=version::$OTP_VERSION" @@ -30,10 +30,13 @@ jobs: run: name: Run checks needs: setup - uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@v1.0.12 + uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@v1.0.18 with: otp-version: ${{ needs.setup.outputs.otp-version }} rebar-version: ${{ needs.setup.outputs.rebar-version }} use-thrift: true thrift-version: ${{ needs.setup.outputs.thrift-version }} run-ct-with-compose: true + use-coveralls: false + cache-version: v7 + upload-coverage: false diff --git a/Makefile b/Makefile index 260a21c..e68010d 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ DEV_IMAGE_TAG = $(TEST_CONTAINER_NAME)-dev DEV_IMAGE_ID = $(file < .image.dev) DOCKER ?= docker -DOCKERCOMPOSE ?= docker-compose +DOCKERCOMPOSE ?= docker compose DOCKERCOMPOSE_W_ENV = DEV_IMAGE_TAG=$(DEV_IMAGE_TAG) $(DOCKERCOMPOSE) -f compose.yaml -f compose.tracing.yaml REBAR ?= rebar3 TEST_CONTAINER_NAME ?= testrunner diff --git a/apps/cds/src/cds.erl b/apps/cds/src/cds.erl index 532e647..84d39ce 100644 --- a/apps/cds/src/cds.erl +++ b/apps/cds/src/cds.erl @@ -92,16 +92,14 @@ enable_health_logging(Check) -> %% -spec start(normal, any()) -> {ok, pid()} | {error, any()}. start(normal, _StartArgs) -> - case supervisor:start_link({local, ?MODULE}, ?MODULE, []) of - {ok, Sup} -> - NSlist = lists:flatten([ - cds_token_storage:get_namespaces(), - cds_card_storage:get_namespaces(), - cds_ident_doc_storage:get_namespaces() - ]), - cds_storage:start(NSlist), - {ok, Sup} - end. + {ok, Sup} = supervisor:start_link({local, ?MODULE}, ?MODULE, []), + NSlist = lists:flatten([ + cds_token_storage:get_namespaces(), + cds_card_storage:get_namespaces(), + cds_ident_doc_storage:get_namespaces() + ]), + cds_storage:start(NSlist), + {ok, Sup}. -spec stop(any()) -> ok. stop(_State) -> diff --git a/apps/cds/src/cds_storage_pg.erl b/apps/cds/src/cds_storage_pg.erl new file mode 100644 index 0000000..63d0166 --- /dev/null +++ b/apps/cds/src/cds_storage_pg.erl @@ -0,0 +1,372 @@ +-module(cds_storage_pg). + +-include_lib("epgsql/include/epgsql.hrl"). + +-export([start/1]). + +-export([put/5]). +-export([get/2]). +-export([update/5]). +-export([delete/2]). +-export([search_by_index_value/5]). +-export([search_by_index_range/6]). +-export([get_keys/3]). + +-type namespace() :: cds_storage:namespace(). +-type key() :: cds_storage:key(). +-type data() :: cds_storage:data(). +-type metadata() :: cds_storage:metadata(). +-type indexes() :: cds_storage:indexes(). +-type index_id() :: cds_storage:index_id(). +-type index_value() :: cds_storage:index_value(). +-type limit() :: cds_storage:limit(). +-type continuation() :: term(). + +-define(EPOCH_DIFF, 62167219200). +-define(POOL, default_pool). + +-spec put(namespace(), key(), data(), metadata(), indexes()) -> ok. +put(NS, Key, Data, Meta, Indexes) -> + put_(NS, Key, Data, Meta, Indexes). + +-spec get(namespace(), key()) -> {ok, {data(), metadata(), indexes()}} | {error, not_found}. +get(NS, Key) -> + get_(NS, Key). + +-spec update(namespace(), key(), data(), metadata(), indexes()) -> ok | {error, not_found}. +update(NS, Key, Data, Meta, Indexes) -> + update_(NS, Key, Data, Meta, Indexes). + +-spec delete(namespace(), key()) -> ok | {error, not_found}. +delete(NS, Key) -> + delete_(NS, Key). + +-spec search_by_index_value( + namespace(), + index_id(), + index_value(), + limit(), + continuation() +) -> {ok, {[key()], continuation()}}. +search_by_index_value(NS, IndexName, IndexValue, Limit, Continuation) -> + get_keys_by_index_range_(NS, IndexName, IndexValue, IndexValue, Limit, Continuation). + +-spec search_by_index_range( + namespace(), + index_id(), + StartValue :: index_value(), + EndValue :: index_value(), + limit(), + continuation() +) -> {ok, {[key()], continuation()}}. +search_by_index_range(NS, IndexName, StartValue, EndValue, Limit, Continuation) -> + get_keys_by_index_range_(NS, IndexName, StartValue, EndValue, Limit, Continuation). + +-spec get_keys(namespace(), limit(), continuation()) -> {ok, {[key()], continuation()}}. +get_keys(NS, Limit, Continuation) -> + get_keys_(NS, Limit, Continuation). + +-spec start([namespace()]) -> ok. +start(NSList) -> + {ok, PgOpts} = application:get_env(cds, ?MODULE), + Databases = #{cds => maps:get(db, PgOpts)}, + Pools = #{ + default_pool => #{ + database => cds, + size => maps:get(pool, PgOpts, 10) + } + }, + _ = application:load(epg_connector), + _ = application:set_env(epg_connector, databases, Databases), + _ = application:set_env(epg_connector, pools, Pools), + _ = maybe_set_env(epg_connector, vault_token_path, maps:get(vault_token_path, PgOpts, undefined)), + _ = maybe_set_env(epg_connector, vault_role, maps:get(vault_role, PgOpts, undefined)), + {ok, _} = application:ensure_all_started(epg_connector), + lists:foreach( + fun(NS) -> + ok = init_ns(NS) + end, + NSList + ), + ok. + +%% + +init_ns(NS) -> + Table = table(NS), + {ok, _, _} = epg_pool:query( + ?POOL, + "CREATE TABLE IF NOT EXISTS " ++ Table ++ + "(key BYTEA PRIMARY KEY, " + " data BYTEA NOT NULL, " + " metadata BYTEA, " + " encoding_key_id INTEGER, " + " hash BYTEA, " + " created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW())" + ), + ok. + +put_(NS, Key, Data, Meta, Indexes) -> + Table = table(NS), + #{ + created_at := CreatedAt, + hash := Hash, + encoding_key_id := EncodingKey + } = parse_indexes(Indexes), + {ok, _} = epg_pool:query( + ?POOL, + "INSERT INTO " ++ Table ++ + " (key, data, metadata, encoding_key_id, hash, created_at)" + " VALUES ($1, $2, $3, $4, $5, $6)" + " ON CONFLICT (key) DO " + " UPDATE SET data = $2, metadata = $3, encoding_key_id = $4, hash = $5, created_at = $6 ", + [Key, Data, define(Meta), EncodingKey, Hash, unixtime_to_datetime(CreatedAt)] + ), + ok. + +get_(NS, Key) -> + Table = table(NS), + case get_query(?POOL, Table, Key) of + {ok, _, []} -> + {error, not_found}; + {ok, Columns, Rows} -> + [Rec] = to_maps(Columns, Rows), + {ok, unmarshal(Rec)} + end. + +update_(NS, Key, Data, Meta, Indexes) -> + Table = table(NS), + Result = epg_pool:transaction( + ?POOL, + fun(Connection) -> + case get_query(Connection, Table, Key) of + {ok, _, []} -> + {error, not_found}; + {ok, Columns, Rows} -> + [Rec] = to_maps(Columns, Rows), + {_, _, OldIndexes} = unmarshal(Rec), + ParsedOldIndexes = parse_indexes(OldIndexes), + ParsedIndexes = parse_indexes(Indexes), + #{ + created_at := CreatedAt, + hash := Hash, + encoding_key_id := EncodingKey + } = maps:merge(ParsedOldIndexes, ParsedIndexes), + update_query(Connection, Table, Key, Data, Meta, EncodingKey, Hash, CreatedAt) + end + end + ), + case Result of + {ok, 1} -> + ok; + {error, _} = Error -> + Error + end. + +delete_(NS, Key) -> + Table = table(NS), + case delete_query(?POOL, Table, Key) of + {ok, 0} -> + {error, not_found}; + {ok, 1} -> + ok + end. + +get_keys_by_index_range_(NS, IndexName, StartValue, EndValue, Limit, Continuation) -> + Table = table(NS), + IndexField = index_field(IndexName), + LimitOffsetExpr = limit_offset(Limit, Continuation), + Result = epg_pool:query( + ?POOL, + "SELECT key FROM " ++ Table ++ + " WHERE " ++ IndexField ++ + " >= $1 " + " AND " ++ IndexField ++ + " <= $2" + " ORDER BY " ++ IndexField ++ LimitOffsetExpr, + [encode_index(IndexField, StartValue), encode_index(IndexField, EndValue)] + ), + case Result of + {ok, _, []} -> + {ok, {[], undefined}}; + {ok, _, RawKeys} -> + Keys = lists:map(fun({Key}) -> Key end, RawKeys), + NextContinuation = shift_continuation(Continuation, erlang:length(Keys)), + {ok, {Keys, NextContinuation}} + end. + +get_keys_(NS, Limit, Continuation) -> + Table = table(NS), + LimitOffsetExpr = limit_offset(Limit, Continuation), + Result = epg_pool:query( + ?POOL, + "SELECT key FROM " ++ Table ++ + " ORDER BY key " ++ LimitOffsetExpr + ), + case Result of + {ok, _, []} -> + {ok, {[], undefined}}; + {ok, _, RawKeys} -> + Keys = lists:map(fun({Key}) -> Key end, RawKeys), + NextContinuation = shift_continuation(Continuation, erlang:length(Keys)), + {ok, {Keys, NextContinuation}} + end. + +get_query(PoolOrConn, Table, Key) -> + epg_pool:query( + PoolOrConn, + "SELECT * FROM " ++ Table ++ " WHERE key = $1", + [Key] + ). + +update_query(PoolOrConn, Table, Key, Data, Meta, EncodingKey, Hash, CreatedAt) -> + epg_pool:query( + PoolOrConn, + "UPDATE " ++ Table ++ + " SET " + " data = $1," + " metadata = $2," + " encoding_key_id = $3," + " hash = $4," + " created_at = $5" + " WHERE key = $6", + [Data, define(Meta), EncodingKey, Hash, unixtime_to_datetime(CreatedAt), Key] + ). + +delete_query(PoolOrConn, Table, Key) -> + epg_pool:query( + PoolOrConn, + "DELETE FROM " ++ Table ++ " WHERE key = $1", + [Key] + ). + +table(NS) -> + unicode:characters_to_list(<<"\"", NS/binary, "\"">>). + +limit_offset(undefined, undefined) -> + " "; +limit_offset(Limit, undefined) -> + " LIMIT " ++ integer_to_list(Limit); +limit_offset(undefined, Continuation) -> + " OFFSET " ++ integer_to_list(Continuation); +limit_offset(Limit, Continuation) -> + " LIMIT " ++ integer_to_list(Limit) ++ " OFFSET " ++ integer_to_list(Continuation). + +encode_index("created_at", Value) -> + unixtime_to_datetime(Value); +encode_index(_, Value) -> + Value. + +shift_continuation(undefined, BatchSize) -> + BatchSize; +shift_continuation(Continuation, BatchSize) -> + Continuation + BatchSize. + +index_field({_, "created_at"}) -> + "created_at"; +index_field({_, "card_data_salted_hash"}) -> + "hash"; +index_field({_, "encoding_key_id"}) -> + "encoding_key_id". + +define(V) -> + define(V, null). + +define(undefined, Default) -> + Default; +define(V, _) -> + V. + +parse_indexes(Indexes) -> + Default = #{ + created_at => null, + hash => null, + encoding_key_id => null + }, + maps:fold( + fun + ({_, "created_at"}, V, Acc) -> + Acc#{created_at => V}; + ({_, "card_data_salted_hash"}, V, Acc) -> + Acc#{hash => V}; + ({_, "encoding_key_id"}, V, Acc) -> + Acc#{encoding_key_id => V} + end, + Default, + maps:from_list(Indexes) + ). + +unmarshal(#{<<"data">> := Data} = Row) -> + Indexes = maps:fold( + fun + (<<"created_at">>, V, Acc) -> + [{{integer_index, "created_at"}, V} | Acc]; + (<<"hash">>, V, Acc) -> + [{{binary_index, "card_data_salted_hash"}, V} | Acc]; + (<<"encoding_key_id">>, V, Acc) -> + [{{integer_index, "encoding_key_id"}, V} | Acc]; + (_, _, Acc) -> + Acc + end, + [], + Row + ), + {Data, maps:get(<<"metadata">>, Row, undefined), Indexes}. + +to_maps(Columns, Rows) -> + to_maps(Columns, Rows, fun(V) -> V end). + +to_maps(Columns, Rows, TransformRowFun) -> + ColNumbers = erlang:length(Columns), + Seq = lists:seq(1, ColNumbers), + lists:map( + fun(Row) -> + Data = lists:foldl( + fun(Pos, Acc) -> + #column{name = Field, type = Type} = lists:nth(Pos, Columns), + maybe_add_field(convert(Type, erlang:element(Pos, Row)), Field, Acc) + end, + #{}, + Seq + ), + TransformRowFun(Data) + end, + Rows + ). + +maybe_add_field(null, _Field, Acc) -> + Acc; +maybe_add_field(Value, Field, Acc) -> + Acc#{Field => Value}. + +%% for reference https://github.com/epgsql/epgsql#data-representation +convert(_Type, null) -> + null; +convert(timestamp, Value) -> + daytime_to_unixtime(Value); +convert(timestamptz, Value) -> + daytime_to_unixtime(Value); +convert(jsonb, Value) -> + jsx:decode(Value, [return_maps]); +convert(json, Value) -> + jsx:decode(Value, [return_maps]); +convert(_Type, Value) -> + Value. + +daytime_to_unixtime({Date, {Hour, Minute, Second}}) when is_float(Second) -> + daytime_to_unixtime({Date, {Hour, Minute, trunc(Second)}}); +daytime_to_unixtime(Daytime) -> + to_unixtime(calendar:datetime_to_gregorian_seconds(Daytime)). + +to_unixtime(Time) when is_integer(Time) -> + Time - ?EPOCH_DIFF. + +unixtime_to_datetime(null) -> + null; +unixtime_to_datetime(TimestampSec) -> + calendar:gregorian_seconds_to_datetime(TimestampSec + ?EPOCH_DIFF). + +maybe_set_env(_App, _Key, undefined) -> + ok; +maybe_set_env(App, Key, Value) -> + application:set_env(App, Key, Value). diff --git a/apps/cds/src/cds_woody_event_handler.erl b/apps/cds/src/cds_woody_event_handler.erl index d669b89..abb6bd2 100644 --- a/apps/cds/src/cds_woody_event_handler.erl +++ b/apps/cds/src/cds_woody_event_handler.erl @@ -14,9 +14,9 @@ %% woody_event_handler behaviour callbacks %% --spec handle_event(Event, RpcId, Meta, Opts) -> ok when +-spec handle_event(Event, RpcID, Meta, Opts) -> ok when Event :: woody_event_handler:event(), - RpcId :: woody:rpc_id() | undefined, + RpcID :: woody:rpc_id() | undefined, Meta :: woody_event_handler:event_meta(), Opts :: woody:options(). handle_event(Event, RpcID, RawMeta, Opts) -> diff --git a/apps/cds/test/cds_card_v2_client.erl b/apps/cds/test/cds_card_v2_client.erl index 52c73d0..1615868 100644 --- a/apps/cds/test/cds_card_v2_client.erl +++ b/apps/cds/test/cds_card_v2_client.erl @@ -65,9 +65,9 @@ get_card_data(Token, RootUrl) -> put_card_and_session(CardData, SessionData, RootUrl) -> case put_card(CardData, RootUrl) of Result when is_map(Result) -> - SessionId = genlib:unique(), - ok = put_session(SessionId, SessionData, RootUrl), - maps:merge(#{session_id => SessionId}, Result); + SessionID = genlib:unique(), + ok = put_session(SessionID, SessionData, RootUrl), + maps:merge(#{session_id => SessionID}, Result); Error -> Error end. @@ -98,9 +98,9 @@ put_card(CardData, RootUrl) -> put_session(SessionID, SessionData, RootUrl) -> call(card_v2, 'PutSession', {SessionID, encode_session_data(SessionData)}, RootUrl). -encode_card_data(#{pan := Pan}) -> +encode_card_data(#{pan := PAN}) -> #cds_PutCardData{ - pan = Pan + pan = PAN }. encode_session_data(undefined) -> @@ -151,13 +151,13 @@ decode_bank_card( decode_card_data( #cds_CardData{ - pan = Pan, + pan = PAN, exp_date = ExpDate, cardholder_name = CardholderName } ) -> DecodedCardData = #{ - pan => Pan, + pan => PAN, exp_date => decode_exp_date(ExpDate), cardholder_name => CardholderName }, diff --git a/apps/cds/test/cds_ct_utils.erl b/apps/cds/test/cds_ct_utils.erl index 8c60780..3067c11 100644 --- a/apps/cds/test/cds_ct_utils.erl +++ b/apps/cds/test/cds_ct_utils.erl @@ -5,6 +5,7 @@ -export([set_riak_storage/1]). -export([set_ets_storage/1]). +-export([set_postgres_storage/1]). -export([store/1]). -export([store/2]). @@ -109,7 +110,12 @@ start_clear(Config) -> -spec stop_clear(config()) -> ok. stop_clear(C) -> - [ok = application:stop(App) || App <- config(apps, C)], + stop_apps(config(apps, C)). + +stop_apps(Apps) when is_list(Apps) -> + [ok = application:stop(App) || App <- Apps], + ok; +stop_apps(_) -> ok. -spec set_riak_storage(config()) -> config(). @@ -143,6 +149,26 @@ set_ets_storage(C) -> ], [{storage_config, StorageConfig} | C]. +-spec set_postgres_storage(config()) -> config(). +set_postgres_storage(C) -> + StorageConfig = [ + {storage, cds_storage_pg}, + {cds_storage_pg, #{ + db => pg_opts(), + pool => 10 + }} + ], + [{storage_config, StorageConfig} | C]. + +pg_opts() -> + #{ + host => "postgres", + port => 5432, + database => "cds", + username => "cds", + password => "password" + }. + -spec store([{any(), any()}]) -> ok. store(KVs) when is_list(KVs) -> [store(Key, Value) || {Key, Value} <- KVs], @@ -194,7 +220,9 @@ clean_storage(CdsEnv) -> cds_storage_riak -> clean_riak_storage(CdsEnv); cds_storage_ets -> - ok + ok; + cds_storage_pg -> + clean_pg_storage() end. clean_riak_storage(CdsEnv) -> @@ -220,6 +248,23 @@ clean_riak_storage(CdsEnv) -> ), ok. +clean_pg_storage() -> + {ok, C} = epgsql:connect(pg_opts()), + NSList = lists:flatten([ + cds_token_storage:get_namespaces(), + cds_card_storage:get_namespaces(), + cds_ident_doc_storage:get_namespaces() + ]), + lists:foreach( + fun(NS) -> + Table = unicode:characters_to_list(<<"\"", NS/binary, "\"">>), + epgsql:equery(C, "TRUNCATE TABLE " ++ Table) + end, + NSList + ), + ok = epgsql:close(C), + ok. + call(Service, Method, Args, RootUrl, Strategy) -> try ExtraOpts = #{ diff --git a/apps/cds/test/cds_storage_api_tests_SUITE.erl b/apps/cds/test/cds_storage_api_tests_SUITE.erl index ccfa92e..9cef6c2 100644 --- a/apps/cds/test/cds_storage_api_tests_SUITE.erl +++ b/apps/cds/test/cds_storage_api_tests_SUITE.erl @@ -90,11 +90,15 @@ all() -> groups() -> [ {all_groups, [], [ + {group, postgres_storage_backend}, {group, riak_storage_backend}, {group, ets_storage_backend}, {group, keyring_errors}, {group, token_check} ]}, + {postgres_storage_backend, [], [ + {group, general_flow} + ]}, {riak_storage_backend, [], [ {group, general_flow}, {group, error_map} @@ -188,6 +192,8 @@ init_per_group(riak_storage_backend, C) -> cds_ct_utils:set_riak_storage(C); init_per_group(ets_storage_backend, C) -> cds_ct_utils:set_ets_storage(C); +init_per_group(postgres_storage_backend, C) -> + cds_ct_utils:set_postgres_storage(C); init_per_group(all_groups, C) -> C; init_per_group(backward_compatibility, C) -> diff --git a/compose.yaml b/compose.yaml index d44cc5d..fe84a07 100644 --- a/compose.yaml +++ b/compose.yaml @@ -18,6 +18,8 @@ services: condition: service_healthy kds: condition: service_healthy + postgres: + condition: service_healthy riakdb: image: docker.io/basho/riak-kv:${RIAK_VERSION} @@ -54,3 +56,20 @@ services: interval: 5s timeout: 1s retries: 20 + + postgres: + image: postgres:15-bookworm + #command: -c 'max_connections=300' + environment: + POSTGRES_DB: "cds" + POSTGRES_USER: "cds" + POSTGRES_PASSWORD: "password" + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U progressor -d progressor_db"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + restart: unless-stopped diff --git a/config/sys.config b/config/sys.config index 92ff1fb..2cc613a 100644 --- a/config/sys.config +++ b/config/sys.config @@ -26,6 +26,7 @@ }}, {keyring_fetch_interval, 60000}, {storage, cds_storage_ets}, + %%% % {storage, cds_storage_riak}, % {cds_storage_riak, #{ % conn_params => #{ @@ -38,6 +39,20 @@ % }, % timeout => 5000 % milliseconds % }}, + %%% + % {storage, cds_storage_pg}, + % {cds_storage_pg, #{ + % db => #{ + % host => "postgres", + % port => 5432, + % database => "cds", + % username => "cds", + % password => "password" + % }, + % pool => 10, + % vault_token_path => "/var/run/secrets/kubernetes.io/serviceaccount/token", + % vault_role => cds + % }}, {session_cleaning, #{ enabled => true, interval => 3000, @@ -97,5 +112,10 @@ {prometheus, [ {collectors, [default]} + ]}, + + {canal, [ + {url, "http://vault"}, + {engine, kvv2} ]} ]. diff --git a/elvis.config b/elvis.config index 161dcc8..583aaad 100644 --- a/elvis.config +++ b/elvis.config @@ -12,6 +12,10 @@ % Too opionated {elvis_style, no_if_expression, disable}, {elvis_style, state_record_and_type, disable}, + {elvis_style, export_used_types, disable}, + {elvis_style, param_pattern_matching, disable}, + {elvis_style, no_throw, disable}, + {elvis_style, no_block_expressions, disable}, {elvis_style, function_naming_convention, #{regex => "^[a-z]([a-z0-9]*_?)*$"}}, {elvis_style, atom_naming_convention, #{regex => "^[a-z]([a-z0-9]*_?)*$"}}, {elvis_style, dont_repeat_yourself, #{min_complexity => 15}} @@ -27,6 +31,9 @@ % Too opionated {elvis_style, no_if_expression, disable}, {elvis_style, state_record_and_type, disable}, + {elvis_style, export_used_types, disable}, + {elvis_style, no_catch_expressions, disable}, + {elvis_style, no_block_expressions, disable}, {elvis_style, function_naming_convention, #{regex => "^[a-z]([a-z0-9]*_?)*$"}}, {elvis_style, atom_naming_convention, #{regex => "^[a-z]([a-z0-9]*_?)*$"}}, % We want to use `ct:pal/2` and friends in test code. @@ -54,6 +61,7 @@ {elvis_text_style, line_length, #{limit => 120}}, {elvis_text_style, no_tabs}, {elvis_text_style, no_trailing_whitespace}, + {elvis_project, no_branch_deps, disable}, %% Temporarily disabled till regex pattern is available {elvis_project, no_deps_master_rebar, disable} ] diff --git a/rebar.config b/rebar.config index 30cad43..4e2d61c 100644 --- a/rebar.config +++ b/rebar.config @@ -30,13 +30,13 @@ %% NOTE %% Pinning to version "1.11.2" from hex here causes constant upgrading and recompilation of the entire project {jose, {git, "https://github.com/potatosalad/erlang-jose.git", {tag, "1.11.2"}}}, - {libdecaf, "2.1.0"}, - {pooler, {git, "https://github.com/seth/pooler.git", {branch, master}}}, - {scrypt, {git, "https://github.com/kpy3/erlscrypt", {tag, "2.0.2"}}}, + {libdecaf, "2.1.1"}, + {pooler, {git, "https://github.com/seth/pooler", {ref, "96e1840"}}}, + {scrypt, {git, "https://github.com/kpy3/erlscrypt", {tag, "2.1.0"}}}, {shamir, {git, "https://github.com/valitydev/shamir.git", {branch, master}}}, - {riakc, {git, "https://github.com/valitydev/riak-erlang-client.git", {branch, develop}}}, - {woody, {git, "https://github.com/valitydev/woody_erlang.git", {branch, master}}}, - {genlib, {git, "https://github.com/valitydev/genlib.git", {branch, master}}}, + {riakc, {git, "https://github.com/valitydev/riak-erlang-client", {branch, "develop"}}}, + {woody, {git, "https://github.com/valitydev/woody_erlang.git", {tag, "v1.1.0"}}}, + {genlib, {git, "https://github.com/valitydev/genlib.git", {tag, "v1.1.0"}}}, {erl_health, {git, "https://github.com/valitydev/erlang-health.git", {branch, master}}}, {cds_proto, {git, "https://github.com/valitydev/cds-proto.git", {branch, master}}}, {tds_proto, {git, "https://github.com/valitydev/tds-proto.git", {branch, master}}}, @@ -45,7 +45,8 @@ {identdocstore_proto, {git, "https://github.com/valitydev/identdocstore-proto.git", {branch, master}}}, {opentelemetry_api, "1.2.1"}, {opentelemetry, "1.3.0"}, - {opentelemetry_exporter, "1.3.0"} + {opentelemetry_exporter, "1.3.0"}, + {epg_connector, {git, "https://github.com/valitydev/epg_connector.git", {branch, "master"}}} ]}. %% XRef checks @@ -64,7 +65,6 @@ % mandatory unmatched_returns, error_handling, - race_conditions, unknown ]}, {plt_apps, all_deps} @@ -85,18 +85,17 @@ {prometheus, "4.8.1"}, {prometheus_cowboy, "0.1.8"}, {logger_logstash_formatter, - {git, "https://github.com/valitydev/logger_logstash_formatter.git", {ref, "08a66a6"}}}, - {iosetopts, {git, "https://github.com/valitydev/iosetopts.git", {ref, "edb445c"}}} + {git, "https://github.com/valitydev/logger_logstash_formatter.git", {ref, "08a66a6"}}} ]}, {relx, [ {release, {cds, "0.1.0"}, [ - iosetopts, {recon, load}, {runtime_tools, load}, {tools, load}, {libdecaf, load}, {opentelemetry, temporary}, {logger_logstash_formatter, load}, + {epg_connector, load}, prometheus, prometheus_cowboy, cds @@ -114,15 +113,16 @@ {cover_enabled, true}, {deps, []}, {dialyzer, [ - {plt_extra_apps, [eunit, common_test]} + {plt_extra_apps, [eunit, common_test, epg_connector, epgsql]} ]} ]} ]}. {project_plugins, [ - {rebar3_lint, "1.0.1"}, - {erlfmt, "1.0.0"}, - {covertool, "2.0.4"} + {rebar3_lint, "3.2.6"}, + {erlfmt, "1.5.0"}, + {covertool, "2.0.7"}, + {rebar3_thrift_compiler, {git, "https://github.com/valitydev/rebar3_thrift_compiler.git", {tag, "0.3.2"}}} ]}. %% Linter config. @@ -141,13 +141,37 @@ ]}. {overrides, [ + {override, rebar3_protobuffs_plugin, [ + {deps, [ + {protobuffs, {git, "https://github.com/basho/erlang_protobuffs.git", {tag, "0.8.2"}}} + ]} + ]}, + {override, protobuffs, [ + {deps, []} + ]}, {override, riakc, [ {erl_opts, [ {d, namespaced_types}, {d, deprecated_19} ]} ]}, + {override, riak_pb, [ + {plugins, [ + {riak_pb_msgcodegen, {git, "https://github.com/tsloughter/riak_pb_msgcodegen", {branch, master}}}, + {rebar3_protobuffs_plugin, {git, "https://github.com/cmkarlsson/rebar3_protobuffs_plugin", {tag, "0.1.1"}}} + ]}, + {provider_hooks, [ + {pre, [ + {compile, {protobuffs, compile}}, + {compile, riak_pb_msgcodegen} + ]} + ]} + ]}, {override, hamcrest, [ {plugins, []} + ]}, + {del, [ + {plugins, [{rebar3_archive_plugin, "0.0.1"}]}, + {plugins, [{rebar3_archive_plugin, "0.0.2"}]} ]} ]}. diff --git a/rebar.lock b/rebar.lock index b461cb3..120ad15 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,6 +1,10 @@ {"1.2.0", [{<<"acceptor_pool">>,{pkg,<<"acceptor_pool">>,<<"1.0.0">>},2}, {<<"cache">>,{pkg,<<"cache">>,<<"2.3.3">>},1}, + {<<"canal">>, + {git,"https://github.com/valitydev/canal", + {ref,"621d3821cd0a6036fee75d8e3b2d17167f3268e4"}}, + 1}, {<<"cds_proto">>, {git,"https://github.com/valitydev/cds-proto.git", {ref,"ed9f9078049ebcd1439d2ada0479fc8f33ccacf1"}}, @@ -10,26 +14,34 @@ {git,"https://github.com/rbkmoney/cg_mon.git", {ref,"5a87a37694e42b6592d3b4164ae54e0e87e24e18"}}, 1}, - {<<"chatterbox">>,{pkg,<<"ts_chatterbox">>,<<"0.13.0">>},2}, + {<<"chatterbox">>,{pkg,<<"ts_chatterbox">>,<<"0.15.1">>},2}, {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.9.0">>},1}, {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.11.0">>},2}, {<<"ctx">>,{pkg,<<"ctx">>,<<"0.6.0">>},2}, + {<<"epg_connector">>, + {git,"https://github.com/valitydev/epg_connector.git", + {ref,"dd93e27c00d492169e8a7bfc38976b911c6e7d05"}}, + 0}, + {<<"epgsql">>, + {git,"https://github.com/epgsql/epgsql.git", + {ref,"7ba52768cf0ea7d084df24d4275a88eef4db13c2"}}, + 1}, {<<"erl_health">>, {git,"https://github.com/valitydev/erlang-health.git", - {ref,"7ffbc855bdbe79e23efad1803b0b185c9ea8d2f1"}}, + {ref,"49716470d0e8dab5e37db55d52dea78001735a3d"}}, 0}, {<<"genlib">>, {git,"https://github.com/valitydev/genlib.git", - {ref,"f6074551d6586998e91a97ea20acb47241254ff3"}}, + {ref,"d2324089afbbd9630e85fac554620f1de0b33dfe"}}, 0}, {<<"gproc">>,{pkg,<<"gproc">>,<<"0.9.0">>},1}, - {<<"grpcbox">>,{pkg,<<"grpcbox">>,<<"0.16.0">>},1}, + {<<"grpcbox">>,{pkg,<<"grpcbox">>,<<"0.17.1">>},1}, {<<"hackney">>,{pkg,<<"hackney">>,<<"1.18.0">>},1}, {<<"hamcrest">>, {git,"https://github.com/basho/hamcrest-erlang.git", {ref,"ad3dbab419762fc2d5821abb88b989da006b85c6"}}, 2}, - {<<"hpack">>,{pkg,<<"hpack_erl">>,<<"0.2.3">>},3}, + {<<"hpack">>,{pkg,<<"hpack_erl">>,<<"0.3.0">>},3}, {<<"identdocstore_proto">>, {git,"https://github.com/valitydev/identdocstore-proto.git", {ref,"7daf193c80bb58b8138f5053c1a93159c448606a"}}, @@ -39,10 +51,11 @@ {git,"https://github.com/potatosalad/erlang-jose.git", {ref,"991649695aaccd92c8effb1c1e88e6159fe8e9a6"}}, 0}, + {<<"jsone">>,{pkg,<<"jsone">>,<<"1.8.0">>},2}, {<<"jsx">>,{pkg,<<"jsx">>,<<"3.1.0">>},0}, - {<<"libdecaf">>,{pkg,<<"libdecaf">>,<<"2.1.0">>},0}, + {<<"libdecaf">>,{pkg,<<"libdecaf">>,<<"2.1.1">>},0}, {<<"metrics">>,{pkg,<<"metrics">>,<<"1.0.1">>},2}, - {<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.2.0">>},2}, + {<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.4.0">>},2}, {<<"msgpack">>, {git,"https://github.com/valitydev/msgpack-erlang", {ref,"9d56647ed77498c7655da39891c4985142697083"}}, @@ -57,10 +70,12 @@ 1}, {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.1">>},2}, {<<"pooler">>, - {git,"https://github.com/seth/pooler.git", - {ref,"9c28fb479f9329e2a1644565a632bc222780f1b7"}}, + {git,"https://github.com/seth/pooler", + {ref,"96e1840b0d67d06b12b14fa3699b13c1d6ebda73"}}, 0}, + {<<"prometheus">>,{pkg,<<"prometheus">>,<<"4.8.1">>},1}, {<<"proper">>,{pkg,<<"proper">>,<<"1.3.0">>},1}, + {<<"quantile_estimator">>,{pkg,<<"quantile_estimator">>,<<"0.2.1">>},2}, {<<"ranch">>,{pkg,<<"ranch">>,<<"1.8.0">>},2}, {<<"recon">>,{pkg,<<"recon">>,<<"2.3.2">>},0}, {<<"riak_pb">>, @@ -68,16 +83,16 @@ {ref,"08771aba2ce4935b715d32d1b92555efdc3da994"}}, 1}, {<<"riakc">>, - {git,"https://github.com/valitydev/riak-erlang-client.git", - {ref,"121a893c429eaa5cab85761771e557f7f15d7659"}}, + {git,"https://github.com/valitydev/riak-erlang-client", + {ref,"c1d364c18e73c9118ba92bbfd538c935800cdabb"}}, 0}, {<<"scoper">>, {git,"https://github.com/valitydev/scoper.git", - {ref,"41a14a558667316998af9f49149ee087ffa8bef2"}}, + {ref,"0e7aa01e9632daa39727edd62d4656ee715b4569"}}, 0}, {<<"scrypt">>, {git,"https://github.com/kpy3/erlscrypt", - {ref,"76ca7ec4b1619dd336a5b024dd5fd2dd67968219"}}, + {ref,"47322ba2a30f7053830cb3309ddc4bde9b99bb92"}}, 0}, {<<"shamir">>, {git,"https://github.com/valitydev/shamir.git", @@ -94,71 +109,77 @@ 0}, {<<"thrift">>, {git,"https://github.com/valitydev/thrift_erlang.git", - {ref,"c280ff266ae1c1906fb0dcee8320bb8d8a4a3c75"}}, + {ref,"3a60e5dc5bbd709495024f26e100b041c3547fd9"}}, 1}, {<<"tls_certificate_check">>, - {pkg,<<"tls_certificate_check">>,<<"1.20.0">>}, + {pkg,<<"tls_certificate_check">>,<<"1.28.0">>}, 1}, - {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.0">>},2}, + {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.1">>},2}, {<<"woody">>, {git,"https://github.com/valitydev/woody_erlang.git", - {ref,"5d46291a6bfcee0bae2a9346a7d927603a909249"}}, + {ref,"cc983a9423325ba1d6a509775eb6ff7ace721539"}}, 0}]}. [ {pkg_hash,[ {<<"acceptor_pool">>, <<"43C20D2ACAE35F0C2BCD64F9D2BDE267E459F0F3FD23DAB26485BF518C281B21">>}, {<<"cache">>, <<"B23A5FE7095445A88412A6E614C933377E0137B44FFED77C9B3FEF1A731A20B2">>}, {<<"certifi">>, <<"D4FB0A6BB20B7C9C3643E22507E42F356AC090A1DCEA9AB99E27E0376D695EBA">>}, - {<<"chatterbox">>, <<"6F059D97BCAA758B8EA6FFFE2B3B81362BD06B639D3EA2BB088335511D691EBF">>}, + {<<"chatterbox">>, <<"5CAC4D15DD7AD61FC3C4415CE4826FC563D4643DEE897A558EC4EA0B1C835C9C">>}, {<<"cowboy">>, <<"865DD8B6607E14CF03282E10E934023A1BD8BE6F6BACF921A7E2A96D800CD452">>}, {<<"cowlib">>, <<"0B9FF9C346629256C42EBE1EEB769A83C6CB771A6EE5960BD110AB0B9B872063">>}, {<<"ctx">>, <<"8FF88B70E6400C4DF90142E7F130625B82086077A45364A78D208ED3ED53C7FE">>}, {<<"gproc">>, <<"853CCB7805E9ADA25D227A157BA966F7B34508F386A3E7E21992B1B484230699">>}, - {<<"grpcbox">>, <<"B83F37C62D6EECA347B77F9B1EC7E9F62231690CDFEB3A31BE07CD4002BA9C82">>}, + {<<"grpcbox">>, <<"6E040AB3EF16FE699FFB513B0EF8E2E896DA7B18931A1EF817143037C454BCCE">>}, {<<"hackney">>, <<"C4443D960BB9FBA6D01161D01CD81173089686717D9490E5D3606644C48D121F">>}, - {<<"hpack">>, <<"17670F83FF984AE6CD74B1C456EDDE906D27FF013740EE4D9EFAA4F1BF999633">>}, + {<<"hpack">>, <<"2461899CC4AB6A0EF8E970C1661C5FC6A52D3C25580BC6DD204F84CE94669926">>}, {<<"idna">>, <<"8A63070E9F7D0C62EB9D9FCB360A7DE382448200FBBD1B106CC96D3D8099DF8D">>}, + {<<"jsone">>, <<"347FF1FA700E182E1F9C5012FA6D737B12C854313B9AE6954CA75D3987D6C06D">>}, {<<"jsx">>, <<"D12516BAA0BB23A59BB35DCCAF02A1BD08243FCBB9EFE24F2D9D056CCFF71268">>}, - {<<"libdecaf">>, <<"26E273443D75420081D4B1F76764492EE3E1D7CD601A2AB1DC8761A6943BFB46">>}, + {<<"libdecaf">>, <<"2D2716C1C2AEE870D15FBF150DCA67210E2A62F11F9126F59019A5EF79F63AD3">>}, {<<"metrics">>, <<"25F094DEA2CDA98213CECC3AEFF09E940299D950904393B2A29D191C346A8486">>}, - {<<"mimerl">>, <<"67E2D3F571088D5CFD3E550C383094B47159F3EEE8FFA08E64106CDF5E981BE3">>}, + {<<"mimerl">>, <<"3882A5CA67FBBE7117BA8947F27643557ADEC38FA2307490C4C4207624CB213B">>}, {<<"opentelemetry">>, <<"988AC3C26ACAC9720A1D4FB8D9DC52E95B45ECFEC2D5B5583276A09E8936BC5E">>}, {<<"opentelemetry_api">>, <<"7B69ED4F40025C005DE0B74FCE8C0549625D59CB4DF12D15C32FE6DC5076FF42">>}, {<<"opentelemetry_exporter">>, <<"1D8809C0D4F4ACF986405F7700ED11992BCBDB6A4915DD11921E80777FFA7167">>}, {<<"opentelemetry_semantic_conventions">>, <<"B67FE459C2938FCAB341CB0951C44860C62347C005ACE1B50F8402576F241435">>}, {<<"parse_trans">>, <<"16328AB840CC09919BD10DAB29E431DA3AF9E9E7E7E6F0089DD5A2D2820011D8">>}, + {<<"prometheus">>, <<"FA76B152555273739C14B06F09F485CF6D5D301FE4E9D31B7FF803D26025D7A0">>}, {<<"proper">>, <<"C1ACD51C51DA17A2FE91D7A6FC6A0C25A6A9849D8DC77093533109D1218D8457">>}, + {<<"quantile_estimator">>, <<"EF50A361F11B5F26B5F16D0696E46A9E4661756492C981F7B2229EF42FF1CD15">>}, {<<"ranch">>, <<"8C7A100A139FD57F17327B6413E4167AC559FBC04CA7448E9BE9057311597A1D">>}, {<<"recon">>, <<"4444C879BE323B1B133EEC5241CB84BD3821EA194C740D75617E106BE4744318">>}, {<<"ssl_verify_fun">>, <<"354C321CF377240C7B8716899E182CE4890C5938111A1296ADD3EC74CF1715DF">>}, - {<<"tls_certificate_check">>, <<"1AC0C53F95E201FEB8D398EF9D764AE74175231289D89F166BA88A7F50CD8E73">>}, - {<<"unicode_util_compat">>, <<"BC84380C9AB48177092F43AC89E4DFA2C6D62B40B8BD132B1059ECC7232F9A78">>}]}, + {<<"tls_certificate_check">>, <<"C39BF21F67C2D124AE905454FAD00F27E625917E8AB1009146E916E1DF6AB275">>}, + {<<"unicode_util_compat">>, <<"A48703A25C170EEDADCA83B11E88985AF08D35F37C6F664D6DCFB106A97782FC">>}]}, {pkg_hash_ext,[ {<<"acceptor_pool">>, <<"0CBCD83FDC8B9AD2EEE2067EF8B91A14858A5883CB7CD800E6FCD5803E158788">>}, {<<"cache">>, <<"44516CE6FA03594D3A2AF025DD3A87BFE711000EB730219E1DDEFC816E0AA2F4">>}, {<<"certifi">>, <<"6AC7EFC1C6F8600B08D625292D4BBF584E14847CE1B6B5C44D983D273E1097EA">>}, - {<<"chatterbox">>, <<"B93D19104D86AF0B3F2566C4CBA2A57D2E06D103728246BA1AC6C3C0FF010AA7">>}, + {<<"chatterbox">>, <<"4F75B91451338BC0DA5F52F3480FA6EF6E3A2AEECFC33686D6B3D0A0948F31AA">>}, {<<"cowboy">>, <<"2C729F934B4E1AA149AFF882F57C6372C15399A20D54F65C8D67BEF583021BDE">>}, {<<"cowlib">>, <<"2B3E9DA0B21C4565751A6D4901C20D1B4CC25CBB7FD50D91D2AB6DD287BC86A9">>}, {<<"ctx">>, <<"A14ED2D1B67723DBEBBE423B28D7615EB0BDCBA6FF28F2D1F1B0A7E1D4AA5FC2">>}, {<<"gproc">>, <<"587E8AF698CCD3504CF4BA8D90F893EDE2B0F58CABB8A916E2BF9321DE3CF10B">>}, - {<<"grpcbox">>, <<"294DF743AE20A7E030889F00644001370A4F7CE0121F3BBDAF13CF3169C62913">>}, + {<<"grpcbox">>, <<"4A3B5D7111DAABC569DC9CBD9B202A3237D81C80BF97212FBC676832CB0CEB17">>}, {<<"hackney">>, <<"9AFCDA620704D720DB8C6A3123E9848D09C87586DC1C10479C42627B905B5C5E">>}, - {<<"hpack">>, <<"06F580167C4B8B8A6429040DF36CC93BBA6D571FAEAEC1B28816523379CBB23A">>}, + {<<"hpack">>, <<"D6137D7079169D8C485C6962DFE261AF5B9EF60FBC557344511C1E65E3D95FB0">>}, {<<"idna">>, <<"92376EB7894412ED19AC475E4A86F7B413C1B9FBB5BD16DCCD57934157944CEA">>}, + {<<"jsone">>, <<"08560B78624A12E0B5E7EC0271EC8CA38EF51F63D84D84843473E14D9B12618C">>}, {<<"jsx">>, <<"0C5CC8FDC11B53CC25CF65AC6705AD39E54ECC56D1C22E4ADB8F5A53FB9427F3">>}, - {<<"libdecaf">>, <<"529D493D2929CEA2EEA2D93464F636A1969756374C58A66AB8F95DD1C0CCFFCB">>}, + {<<"libdecaf">>, <<"DEE36DC9A73CC2384B0F9A53CE7F76C29A1344F28C459678ADD231CB8A2C4C61">>}, {<<"metrics">>, <<"69B09ADDDC4F74A40716AE54D140F93BEB0FB8978D8636EADED0C31B6F099F16">>}, - {<<"mimerl">>, <<"F278585650AA581986264638EBF698F8BB19DF297F66AD91B18910DFC6E19323">>}, + {<<"mimerl">>, <<"13AF15F9F68C65884ECCA3A3891D50A7B57D82152792F3E19D88650AA126B144">>}, {<<"opentelemetry">>, <<"8E09EDC26AAD11161509D7ECAD854A3285D88580F93B63B0B1CF0BAC332BFCC0">>}, {<<"opentelemetry_api">>, <<"6D7A27B7CAD2AD69A09CABF6670514CAFCEC717C8441BEB5C96322BAC3D05350">>}, {<<"opentelemetry_exporter">>, <<"2B40007F509D38361744882FD060A8841AF772AB83BB542AA5350908B303AD65">>}, {<<"opentelemetry_semantic_conventions">>, <<"D61FA1F5639EE8668D74B527E6806E0503EFC55A42DB7B5F39939D84C07D6895">>}, {<<"parse_trans">>, <<"07CD9577885F56362D414E8C4C4E6BDF10D43A8767ABB92D24CBE8B24C54888B">>}, + {<<"prometheus">>, <<"6EDFBE928D271C7F657A6F2C46258738086584BD6CAE4A000B8B9A6009BA23A5">>}, {<<"proper">>, <<"4AA192FCCDDD03FDBE50FEF620BE9D4D2F92635B54F55FB83AEC185994403CBC">>}, + {<<"quantile_estimator">>, <<"282A8A323CA2A845C9E6F787D166348F776C1D4A41EDE63046D72D422E3DA946">>}, {<<"ranch">>, <<"49FBCFD3682FAB1F5D109351B61257676DA1A2FDBE295904176D5E521A2DDFE5">>}, {<<"recon">>, <<"A6FCFBDABE1AEF9119B871304846B1CE282D9A4124BCDDAFC7616C1A256C7596">>}, {<<"ssl_verify_fun">>, <<"FE4C190E8F37401D30167C8C405EDA19469F34577987C76DDE613E838BBC67F8">>}, - {<<"tls_certificate_check">>, <<"AB57B74B1A63DC5775650699A3EC032EC0065005EFF1F020818742B7312A8426">>}, - {<<"unicode_util_compat">>, <<"25EEE6D67DF61960CF6A794239566599B09E17E668D3700247BC498638152521">>}]} + {<<"tls_certificate_check">>, <<"3AB058C3F9457FFFCA916729587415F0DDC822048A0E5B5E2694918556D92DF1">>}, + {<<"unicode_util_compat">>, <<"B3A917854CE3AE233619744AD1E0102E05673136776FB2FA76234F3E03B23642">>}]} ].