diff --git a/apps/hellgate/src/hg_invoice_payment.erl b/apps/hellgate/src/hg_invoice_payment.erl index 0670b4d3..c02fea77 100644 --- a/apps/hellgate/src/hg_invoice_payment.erl +++ b/apps/hellgate/src/hg_invoice_payment.erl @@ -271,7 +271,7 @@ get_payment(#st{payment = Payment}) -> get_risk_score(#st{risk_score = RiskScore}) -> RiskScore. --spec get_route(st()) -> route(). +-spec get_route(st()) -> route() | undefined. get_route(#st{routes = []}) -> undefined; get_route(#st{routes = [Route | _AttemptedRoutes]}) -> @@ -2048,14 +2048,15 @@ process_routing(Action, St) -> end. run_routing_decision_pipeline(Ctx0, VS, St) -> + %% NOTE Since this is routing step then current attempt is not yet + %% accounted for in `St`. + NewIter = get_iter(St) + 1, hg_routing_ctx:pipeline( Ctx0, [ fun(Ctx) -> filter_attempted_routes(Ctx, St) end, - %% Since this is routing step then current attempt is not yet - %% accounted for in `St`. - fun(Ctx) -> filter_routes_with_limit_hold(Ctx, VS, get_iter(St) + 1, St) end, - fun(Ctx) -> filter_routes_by_limit_overflow(Ctx, VS, St) end, + fun(Ctx) -> filter_routes_with_limit_hold(Ctx, VS, NewIter, St) end, + fun(Ctx) -> filter_routes_by_limit_overflow(Ctx, VS, NewIter, St) end, fun(Ctx) -> hg_routing:filter_by_blacklist(Ctx, build_blacklist_context(St)) end, fun hg_routing:filter_by_critical_provider_status/1, fun hg_routing:choose_route_with_ctx/1 @@ -2601,8 +2602,8 @@ filter_routes_with_limit_hold(Ctx0, VS, Iter, St) -> Ctx1 = reject_routes(limit_misconfiguration, RejectedRoutes, Ctx0), hg_routing_ctx:stash_current_candidates(Ctx1). -filter_routes_by_limit_overflow(Ctx0, VS, St) -> - {_Routes, RejectedRoutes, Limits} = get_limit_overflow_routes(hg_routing_ctx:candidates(Ctx0), VS, St), +filter_routes_by_limit_overflow(Ctx0, VS, Iter, St) -> + {_Routes, RejectedRoutes, Limits} = get_limit_overflow_routes(hg_routing_ctx:candidates(Ctx0), VS, Iter, St), Ctx1 = hg_routing_ctx:stash_route_limits(Limits, Ctx0), reject_routes(limit_overflow, RejectedRoutes, Ctx1). @@ -2613,7 +2614,7 @@ reject_routes(GroupReason, RejectedRoutes, Ctx) -> RejectedRoutes ). -get_limit_overflow_routes(Routes, VS, St) -> +get_limit_overflow_routes(Routes, VS, Iter, St) -> Opts = get_opts(St), Revision = get_payment_revision(St), Payment = get_payment(St), @@ -2623,7 +2624,7 @@ get_limit_overflow_routes(Routes, VS, St) -> PaymentRoute = hg_route:to_payment_route(Route), ProviderTerms = hg_routing:get_payment_terms(PaymentRoute, VS, Revision), TurnoverLimits = get_turnover_limits(ProviderTerms), - case hg_limiter:check_limits(TurnoverLimits, Invoice, Payment, PaymentRoute) of + case hg_limiter:check_limits(TurnoverLimits, Invoice, Payment, PaymentRoute, Iter) of {ok, Limits} -> {[Route | RoutesNoOverflowIn], RejectedIn, LimitsIn#{PaymentRoute => Limits}}; {error, {limit_overflow, IDs, Limits}} -> @@ -2656,8 +2657,10 @@ commit_shop_limits(Opts, St) -> check_shop_limits(Opts, St) -> Payment = get_payment(St), Invoice = get_invoice(Opts), + Party = get_party(Opts), + Shop = get_shop(Opts), TurnoverLimits = get_shop_turnover_limits(get_shop(Opts)), - hg_limiter:check_shop_limits(TurnoverLimits, Invoice, Payment). + hg_limiter:check_shop_limits(TurnoverLimits, Party, Shop, Invoice, Payment). rollback_shop_limits(Opts, St, Flags) -> Payment = get_payment(St), @@ -2694,7 +2697,7 @@ hold_limit_routes(Routes0, VS, Iter, St) -> ProviderTerms = hg_routing:get_payment_terms(PaymentRoute, VS, Revision), TurnoverLimits = get_turnover_limits(ProviderTerms), try - ok = hg_limiter:hold_payment_limits(TurnoverLimits, PaymentRoute, Iter, Invoice, Payment), + ok = hg_limiter:hold_payment_limits(TurnoverLimits, Invoice, Payment, PaymentRoute, Iter), {[Route | LimitHeldRoutes], RejectedRoutes} catch error:(#limiter_InvalidOperationCurrency{} = LimiterError) -> @@ -2725,7 +2728,7 @@ rollback_payment_limits(Routes, Iter, St, Flags) -> fun(Route) -> ProviderTerms = hg_routing:get_payment_terms(Route, VS, Revision), TurnoverLimits = get_turnover_limits(ProviderTerms), - ok = hg_limiter:rollback_payment_limits(TurnoverLimits, Route, Iter, Invoice, Payment, Flags) + ok = hg_limiter:rollback_payment_limits(TurnoverLimits, Invoice, Payment, Route, Iter, Flags) end, Routes ). @@ -2749,7 +2752,7 @@ rollback_broken_payment_limits(St) -> [], Values ), - ok = hg_limiter:rollback_payment_limits(TurnoverLimits, Route, Iter, Invoice, Payment, [ + ok = hg_limiter:rollback_payment_limits(TurnoverLimits, Invoice, Payment, Route, Iter, [ ignore_business_error ]) end, @@ -2777,7 +2780,7 @@ commit_payment_limits(#st{capture_data = CaptureData} = St) -> ProviderTerms = get_provider_terms(St, Revision), TurnoverLimits = get_turnover_limits(ProviderTerms), Iter = get_iter(St), - hg_limiter:commit_payment_limits(TurnoverLimits, Route, Iter, Invoice, Payment, CapturedCash). + hg_limiter:commit_payment_limits(TurnoverLimits, Invoice, Payment, Route, Iter, CapturedCash). commit_payment_cashflow(St) -> Plan = get_cashflow_plan(St), @@ -3571,12 +3574,22 @@ get_limit_values(St) -> Ctx = build_routing_context(PaymentInstitution, VS, Revision, St), Payment = get_payment(St), Invoice = get_invoice(get_opts(St)), + %% NOTE If event 'route_changed' didn't occur, then there may be + %% no route yet, however this must be accounted as first iteration + %% of routing attempt. + Iter = + case get_route(St) of + undefined -> 1; + _ -> get_iter(St) + end, lists:foldl( fun(Route, Acc) -> PaymentRoute = hg_route:to_payment_route(Route), ProviderTerms = hg_routing:get_payment_terms(PaymentRoute, VS, Revision), TurnoverLimits = get_turnover_limits(ProviderTerms), - TurnoverLimitValues = hg_limiter:get_limit_values(TurnoverLimits, Invoice, Payment, PaymentRoute), + TurnoverLimitValues = hg_limiter:get_limit_values( + TurnoverLimits, Invoice, Payment, PaymentRoute, Iter + ), Acc#{PaymentRoute => TurnoverLimitValues} end, #{}, diff --git a/apps/hellgate/src/hg_invoice_registered_payment.erl b/apps/hellgate/src/hg_invoice_registered_payment.erl index 0c7aedf3..6ef00630 100644 --- a/apps/hellgate/src/hg_invoice_registered_payment.erl +++ b/apps/hellgate/src/hg_invoice_registered_payment.erl @@ -189,7 +189,7 @@ hold_payment_limits(Invoice, Payment, St) -> Route = hg_invoice_payment:get_route(St), TurnoverLimits = get_turnover_limits(Payment, Route, St), Iter = hg_invoice_payment:get_iter(St), - hg_limiter:hold_payment_limits(TurnoverLimits, Route, Iter, Invoice, Payment). + hg_limiter:hold_payment_limits(TurnoverLimits, Invoice, Payment, Route, Iter). get_turnover_limits(Payment, Route, St) -> Route = hg_invoice_payment:get_route(St), diff --git a/apps/hellgate/src/hg_limiter.erl b/apps/hellgate/src/hg_limiter.erl index 786ae093..e05601cc 100644 --- a/apps/hellgate/src/hg_limiter.erl +++ b/apps/hellgate/src/hg_limiter.erl @@ -24,8 +24,8 @@ -export_type([turnover_limit_value/0]). -export([get_turnover_limits/1]). --export([check_limits/4]). --export([check_shop_limits/3]). +-export([check_limits/5]). +-export([check_shop_limits/5]). -export([hold_payment_limits/5]). -export([hold_shop_limits/5]). -export([hold_refund_limits/5]). @@ -35,7 +35,7 @@ -export([rollback_payment_limits/6]). -export([rollback_shop_limits/6]). -export([rollback_refund_limits/5]). --export([get_limit_values/4]). +-export([get_limit_values/5]). -define(route(ProviderRef, TerminalRef), #domain_PaymentRoute{ provider = ProviderRef, @@ -58,29 +58,55 @@ get_turnover_limits({value, Limits}) -> get_turnover_limits(Ambiguous) -> error({misconfiguration, {'Could not reduce selector to a value', Ambiguous}}). --spec get_limit_values([turnover_limit()], invoice(), payment(), route()) -> [turnover_limit_value()]. -get_limit_values(TurnoverLimits, Invoice, Payment, Route) -> +-spec get_limit_values([turnover_limit()], invoice(), payment(), route(), pos_integer()) -> [turnover_limit_value()]. +get_limit_values(TurnoverLimits, Invoice, Payment, Route, Iter) -> Context = gen_limit_context(Invoice, Payment, Route), + get_limit_values(Context, TurnoverLimits, make_route_operation_segments(Invoice, Payment, Route, Iter)). + +make_route_operation_segments(Invoice, Payment, ?route(ProviderRef, TerminalRef), Iter) -> + [ + genlib:to_binary(get_provider_id(ProviderRef)), + genlib:to_binary(get_terminal_id(TerminalRef)), + get_invoice_id(Invoice), + get_payment_id(Payment), + integer_to_binary(Iter) + ]. + +get_limit_values(Context, TurnoverLimits, OperationIdSegments) -> + {LegacyTurnoverLimits, BatchTurnoverLimits} = split_turnover_limits_by_available_limiter_api(TurnoverLimits), + get_legacy_limit_values(Context, LegacyTurnoverLimits) ++ + get_batch_limit_values(Context, BatchTurnoverLimits, OperationIdSegments). + +get_legacy_limit_values(Context, TurnoverLimits) -> lists:foldl( fun(TurnoverLimit, Acc) -> #domain_TurnoverLimit{id = LimitID, domain_revision = Version} = TurnoverLimit, Clock = get_latest_clock(), Limit = hg_limiter_client:get(LimitID, Version, Clock, Context), - #limiter_Limit{ - amount = LimiterAmount - } = Limit, + #limiter_Limit{amount = LimiterAmount} = Limit, [#payproc_TurnoverLimitValue{limit = TurnoverLimit, value = LimiterAmount} | Acc] end, [], TurnoverLimits ). --spec check_limits([turnover_limit()], invoice(), payment(), route()) -> +get_batch_limit_values(_Context, [], _OperationIdSegments) -> + []; +get_batch_limit_values(Context, TurnoverLimits, OperationIdSegments) -> + {LimitRequest, TurnoverLimitsMap} = prepare_limit_request(TurnoverLimits, OperationIdSegments), + lists:map( + fun(#limiter_Limit{id = Id, amount = Amount}) -> + #payproc_TurnoverLimitValue{limit = maps:get(Id, TurnoverLimitsMap), value = Amount} + end, + hg_limiter_client:get_batch(LimitRequest, Context) + ). + +-spec check_limits([turnover_limit()], invoice(), payment(), route(), pos_integer()) -> {ok, [turnover_limit_value()]} | {error, {limit_overflow, [binary()], [turnover_limit_value()]}}. -check_limits(TurnoverLimits, Invoice, Payment, Route) -> +check_limits(TurnoverLimits, Invoice, Payment, Route, Iter) -> Context = gen_limit_context(Invoice, Payment, Route), - {ok, Limits} = gather_limits(TurnoverLimits, Context, []), + Limits = get_limit_values(Context, TurnoverLimits, make_route_operation_segments(Invoice, Payment, Route, Iter)), try ok = check_limits_(Limits, Context), {ok, Limits} @@ -90,21 +116,28 @@ check_limits(TurnoverLimits, Invoice, Payment, Route) -> {error, {limit_overflow, IDs, Limits}} end. --spec check_shop_limits([turnover_limit()], invoice(), payment()) -> +-spec check_shop_limits([turnover_limit()], party(), shop(), invoice(), payment()) -> ok | {error, {limit_overflow, [binary()]}}. -check_shop_limits(TurnoverLimits, Invoice, Payment) -> +check_shop_limits(TurnoverLimits, Party, Shop, Invoice, Payment) -> Context = gen_limit_shop_context(Invoice, Payment), - {ok, Limits} = gather_limits(TurnoverLimits, Context, []), + Limits = get_limit_values(Context, TurnoverLimits, make_shop_operation_segments(Party, Shop, Invoice, Payment)), try - ok = check_limits_(Limits, Context), - ok + check_limits_(Limits, Context) catch throw:limit_overflow -> IDs = [T#domain_TurnoverLimit.id || T <- TurnoverLimits], {error, {limit_overflow, IDs}} end. +make_shop_operation_segments(?party(PartyID), ?shop(ShopID), Invoice, Payment) -> + [ + PartyID, + ShopID, + get_invoice_id(Invoice), + get_payment_id(Payment) + ]. + check_limits_([], _) -> ok; check_limits_([TurnoverLimitValue | TLVs], Context) -> @@ -127,67 +160,110 @@ check_limits_([TurnoverLimitValue | TLVs], Context) -> throw(limit_overflow) end. -gather_limits([], _Context, Acc) -> - {ok, Acc}; -gather_limits([T | TurnoverLimits], Context, Acc) -> - #domain_TurnoverLimit{id = LimitID, domain_revision = Version} = T, - Clock = get_latest_clock(), - #limiter_Limit{amount = Amount} = hg_limiter_client:get(LimitID, Version, Clock, Context), - TurnoverLimitValue = #payproc_TurnoverLimitValue{limit = T, value = Amount}, - gather_limits(TurnoverLimits, Context, [TurnoverLimitValue | Acc]). +-spec hold_payment_limits([turnover_limit()], invoice(), payment(), route(), pos_integer()) -> ok. +hold_payment_limits(TurnoverLimits, Invoice, Payment, Route, Iter) -> + Context = gen_limit_context(Invoice, Payment, Route), + {LegacyTurnoverLimits, BatchTurnoverLimits} = split_turnover_limits_by_available_limiter_api(TurnoverLimits), + ok = legacy_hold_payment_limits(Context, LegacyTurnoverLimits, Invoice, Payment, Route, Iter), + ok = batch_hold_limits(Context, BatchTurnoverLimits, make_route_operation_segments(Invoice, Payment, Route, Iter)). --spec hold_payment_limits([turnover_limit()], route(), pos_integer(), invoice(), payment()) -> ok. -hold_payment_limits(TurnoverLimits, Route, Iter, Invoice, Payment) -> +legacy_hold_payment_limits(Context, TurnoverLimits, Invoice, Payment, Route, Iter) -> ChangeIDs = [construct_payment_change_id(Route, Iter, Invoice, Payment)], LimitChanges = gen_limit_changes(TurnoverLimits, ChangeIDs), - Context = gen_limit_context(Invoice, Payment, Route), hold(LimitChanges, get_latest_clock(), Context). +batch_hold_limits(_Context, [], _OperationIdSegments) -> + ok; +batch_hold_limits(Context, TurnoverLimits, OperationIdSegments) -> + {LimitRequest, _} = prepare_limit_request(TurnoverLimits, OperationIdSegments), + _ = hg_limiter_client:hold_batch(LimitRequest, Context), + ok. + -spec hold_shop_limits([turnover_limit()], party(), shop(), invoice(), payment()) -> ok. hold_shop_limits(TurnoverLimits, Party, Shop, Invoice, Payment) -> + Context = gen_limit_shop_context(Invoice, Payment), + {LegacyTurnoverLimits, BatchTurnoverLimits} = split_turnover_limits_by_available_limiter_api(TurnoverLimits), + ok = legacy_hold_shop_limits(Context, LegacyTurnoverLimits, Party, Shop, Invoice, Payment), + ok = batch_hold_limits(Context, BatchTurnoverLimits, make_shop_operation_segments(Party, Shop, Invoice, Payment)). + +legacy_hold_shop_limits(Context, TurnoverLimits, Party, Shop, Invoice, Payment) -> ChangeIDs = [construct_shop_change_id(Party, Shop, Invoice, Payment)], LimitChanges = gen_limit_changes(TurnoverLimits, ChangeIDs), - Context = gen_limit_shop_context(Invoice, Payment), hold(LimitChanges, get_latest_clock(), Context). -spec hold_refund_limits([turnover_limit()], invoice(), payment(), refund(), route()) -> ok. hold_refund_limits(TurnoverLimits, Invoice, Payment, Refund, Route) -> + Context = gen_limit_refund_context(Invoice, Payment, Refund, Route), + {LegacyTurnoverLimits, BatchTurnoverLimits} = split_turnover_limits_by_available_limiter_api(TurnoverLimits), + ok = legacy_hold_refund_limits(Context, LegacyTurnoverLimits, Invoice, Payment, Refund), + ok = batch_hold_limits(Context, BatchTurnoverLimits, make_refund_operation_segments(Invoice, Payment, Refund)). + +make_refund_operation_segments(Invoice, Payment, Refund) -> + [ + get_invoice_id(Invoice), + get_payment_id(Payment), + {refund_session, get_refund_id(Refund)} + ]. + +legacy_hold_refund_limits(Context, TurnoverLimits, Invoice, Payment, Refund) -> ChangeIDs = [construct_refund_change_id(Invoice, Payment, Refund)], LimitChanges = gen_limit_changes(TurnoverLimits, ChangeIDs), - Context = gen_limit_refund_context(Invoice, Payment, Refund, Route), hold(LimitChanges, get_latest_clock(), Context). --spec commit_payment_limits([turnover_limit()], route(), pos_integer(), invoice(), payment(), cash() | undefined) -> ok. -commit_payment_limits(TurnoverLimits, Route, Iter, Invoice, Payment, CapturedCash) -> +-spec commit_payment_limits([turnover_limit()], invoice(), payment(), route(), pos_integer(), cash() | undefined) -> ok. +commit_payment_limits(TurnoverLimits, Invoice, Payment, Route, Iter, CapturedCash) -> + Context = gen_limit_context(Invoice, Payment, Route, CapturedCash), + {LegacyTurnoverLimits, BatchTurnoverLimits} = split_turnover_limits_by_available_limiter_api(TurnoverLimits), + Clock = get_latest_clock(), + ok = legacy_commit_payment_limits(Clock, Context, LegacyTurnoverLimits, Invoice, Payment, Route, Iter), + OperationIdSegments = make_route_operation_segments(Invoice, Payment, Route, Iter), + ok = batch_commit_limits(Context, BatchTurnoverLimits, OperationIdSegments), + ok = log_limit_changes(TurnoverLimits, Clock, Context). + +batch_commit_limits(_Context, [], _OperationIdSegments) -> + ok; +batch_commit_limits(Context, TurnoverLimits, OperationIdSegments) -> + {LimitRequest, _} = prepare_limit_request(TurnoverLimits, OperationIdSegments), + hg_limiter_client:commit_batch(LimitRequest, Context). + +legacy_commit_payment_limits(Clock, Context, TurnoverLimits, Invoice, Payment, Route, Iter) -> ChangeIDs = [ construct_payment_change_id(Route, Iter, Invoice, Payment), construct_payment_change_id(Route, Iter, Invoice, Payment, legacy) ], LimitChanges = gen_limit_changes(TurnoverLimits, ChangeIDs), - Context = gen_limit_context(Invoice, Payment, Route, CapturedCash), - Clock = get_latest_clock(), - ok = commit(LimitChanges, Clock, Context), - ok = log_limit_changes(TurnoverLimits, Clock, Context). + commit(LimitChanges, Clock, Context). -spec commit_shop_limits([turnover_limit()], party(), shop(), invoice(), payment()) -> ok. commit_shop_limits(TurnoverLimits, Party, Shop, Invoice, Payment) -> - ChangeIDs = [construct_shop_change_id(Party, Shop, Invoice, Payment)], - LimitChanges = gen_limit_changes(TurnoverLimits, ChangeIDs), Context = gen_limit_shop_context(Invoice, Payment), + {LegacyTurnoverLimits, BatchTurnoverLimits} = split_turnover_limits_by_available_limiter_api(TurnoverLimits), Clock = get_latest_clock(), - ok = commit(LimitChanges, Clock, Context), - ok = log_limit_changes(TurnoverLimits, Clock, Context), - ok. + ok = legacy_commit_shop_limits(Clock, Context, LegacyTurnoverLimits, Party, Shop, Invoice, Payment), + OperationIdSegments = make_shop_operation_segments(Party, Shop, Invoice, Payment), + ok = batch_commit_limits(Context, BatchTurnoverLimits, OperationIdSegments), + ok = log_limit_changes(TurnoverLimits, Clock, Context). + +legacy_commit_shop_limits(Clock, Context, TurnoverLimits, Party, Shop, Invoice, Payment) -> + ChangeIDs = [construct_shop_change_id(Party, Shop, Invoice, Payment)], + LimitChanges = gen_limit_changes(TurnoverLimits, ChangeIDs), + ok = commit(LimitChanges, Clock, Context). -spec commit_refund_limits([turnover_limit()], invoice(), payment(), refund(), route()) -> ok. commit_refund_limits(TurnoverLimits, Invoice, Payment, Refund, Route) -> - ChangeIDs = [construct_refund_change_id(Invoice, Payment, Refund)], - LimitChanges = gen_limit_changes(TurnoverLimits, ChangeIDs), Context = gen_limit_refund_context(Invoice, Payment, Refund, Route), + {LegacyTurnoverLimits, BatchTurnoverLimits} = split_turnover_limits_by_available_limiter_api(TurnoverLimits), Clock = get_latest_clock(), - ok = commit(LimitChanges, Clock, Context), + ok = legacy_commit_refund_limits(Clock, Context, LegacyTurnoverLimits, Invoice, Payment, Refund), + OperationIdSegments = make_refund_operation_segments(Invoice, Payment, Refund), + ok = batch_commit_limits(Context, BatchTurnoverLimits, OperationIdSegments), ok = log_limit_changes(TurnoverLimits, Clock, Context). +legacy_commit_refund_limits(Clock, Context, TurnoverLimits, Invoice, Payment, Refund) -> + ChangeIDs = [construct_refund_change_id(Invoice, Payment, Refund)], + LimitChanges = gen_limit_changes(TurnoverLimits, ChangeIDs), + commit(LimitChanges, Clock, Context). + %% @doc This function supports flags that can change reaction behaviour to %% limiter response: %% @@ -196,30 +272,54 @@ commit_refund_limits(TurnoverLimits, Invoice, Payment, Refund, Route) -> %% %% - `ignore_not_found` -- does not raise error if limiter won't be able to %% find according posting plan in accountant service --spec rollback_payment_limits([turnover_limit()], route(), pos_integer(), invoice(), payment(), [handling_flag()]) -> +-spec rollback_payment_limits([turnover_limit()], invoice(), payment(), route(), pos_integer(), [handling_flag()]) -> ok. -rollback_payment_limits(TurnoverLimits, Route, Iter, Invoice, Payment, Flags) -> +rollback_payment_limits(TurnoverLimits, Invoice, Payment, Route, Iter, Flags) -> + Context = gen_limit_context(Invoice, Payment, Route), + {LegacyTurnoverLimits, BatchTurnoverLimits} = split_turnover_limits_by_available_limiter_api(TurnoverLimits), + ok = legacy_rollback_payment_limits(Context, LegacyTurnoverLimits, Invoice, Payment, Route, Iter, Flags), + OperationIdSegments = make_route_operation_segments(Invoice, Payment, Route, Iter), + ok = batch_rollback_limits(Context, BatchTurnoverLimits, OperationIdSegments). + +batch_rollback_limits(_Context, [], _OperationIdSegments) -> + ok; +batch_rollback_limits(Context, TurnoverLimits, OperationIdSegments) -> + {LimitRequest, _} = prepare_limit_request(TurnoverLimits, OperationIdSegments), + hg_limiter_client:rollback_batch(LimitRequest, Context). + +legacy_rollback_payment_limits(Context, TurnoverLimits, Invoice, Payment, Route, Iter, Flags) -> ChangeIDs = [ construct_payment_change_id(Route, Iter, Invoice, Payment), construct_payment_change_id(Route, Iter, Invoice, Payment, legacy) ], LimitChanges = gen_limit_changes(TurnoverLimits, ChangeIDs), - Context = gen_limit_context(Invoice, Payment, Route), rollback(LimitChanges, get_latest_clock(), Context, Flags). -spec rollback_shop_limits([turnover_limit()], party(), shop(), invoice(), payment(), [handling_flag()]) -> ok. rollback_shop_limits(TurnoverLimits, Party, Shop, Invoice, Payment, Flags) -> + Context = gen_limit_shop_context(Invoice, Payment), + {LegacyTurnoverLimits, BatchTurnoverLimits} = split_turnover_limits_by_available_limiter_api(TurnoverLimits), + ok = legacy_rollback_shop_limits(Context, LegacyTurnoverLimits, Party, Shop, Invoice, Payment, Flags), + OperationIdSegments = make_shop_operation_segments(Party, Shop, Invoice, Payment), + ok = batch_rollback_limits(Context, BatchTurnoverLimits, OperationIdSegments). + +legacy_rollback_shop_limits(Context, TurnoverLimits, Party, Shop, Invoice, Payment, Flags) -> ChangeIDs = [construct_shop_change_id(Party, Shop, Invoice, Payment)], LimitChanges = gen_limit_changes(TurnoverLimits, ChangeIDs), - Context = gen_limit_shop_context(Invoice, Payment), rollback(LimitChanges, get_latest_clock(), Context, Flags). -spec rollback_refund_limits([turnover_limit()], invoice(), payment(), refund(), route()) -> ok. rollback_refund_limits(TurnoverLimits, Invoice, Payment, Refund, Route) -> + Context = gen_limit_refund_context(Invoice, Payment, Refund, Route), + {LegacyTurnoverLimits, BatchTurnoverLimits} = split_turnover_limits_by_available_limiter_api(TurnoverLimits), + ok = legacy_rollback_refund_limits(Context, LegacyTurnoverLimits, Invoice, Payment, Refund), + OperationIdSegments = make_refund_operation_segments(Invoice, Payment, Refund), + ok = batch_rollback_limits(Context, BatchTurnoverLimits, OperationIdSegments). + +legacy_rollback_refund_limits(Context, TurnoverLimits, Invoice, Payment, Refund) -> ChangeIDs = [construct_refund_change_id(Invoice, Payment, Refund)], LimitChanges = gen_limit_changes(TurnoverLimits, ChangeIDs), - Context = gen_limit_refund_context(Invoice, Payment, Refund, Route), rollback(LimitChanges, get_latest_clock(), Context, []). -spec hold([change_queue()], hg_limiter_client:clock(), hg_limiter_client:context()) -> ok. @@ -448,3 +548,23 @@ maybe_route_context(#base_Route{provider = Provider, terminal = Terminal}) -> provider_id => Provider#domain_ProviderRef.id, terminal_id => Terminal#domain_TerminalRef.id }. + +split_turnover_limits_by_available_limiter_api(TurnoverLimits) -> + lists:partition(fun(#domain_TurnoverLimit{domain_revision = V}) -> V =:= undefined end, TurnoverLimits). + +prepare_limit_request(TurnoverLimits, IdSegments) -> + {TurnoverLimitsIdList, LimitChanges} = lists:unzip( + lists:map( + fun(TurnoverLimit = #domain_TurnoverLimit{id = Id, domain_revision = DomainRevision}) -> + {{Id, TurnoverLimit}, #limiter_LimitChange{id = Id, version = DomainRevision}} + end, + TurnoverLimits + ) + ), + OperationId = make_operation_id(IdSegments), + LimitRequest = #limiter_LimitRequest{operation_id = OperationId, limit_changes = LimitChanges}, + TurnoverLimitsMap = maps:from_list(TurnoverLimitsIdList), + {LimitRequest, TurnoverLimitsMap}. + +make_operation_id(IdSegments) -> + hg_utils:construct_complex_id([<<"limiter">>, <<"batch-request">>] ++ IdSegments). diff --git a/apps/hellgate/src/hg_limiter_client.erl b/apps/hellgate/src/hg_limiter_client.erl index bbd32421..a8569e19 100644 --- a/apps/hellgate/src/hg_limiter_client.erl +++ b/apps/hellgate/src/hg_limiter_client.erl @@ -8,12 +8,19 @@ -export([commit/3]). -export([rollback/3]). +-export([get_values/2]). +-export([get_batch/2]). +-export([hold_batch/2]). +-export([commit_batch/2]). +-export([rollback_batch/2]). + -type limit() :: limproto_limiter_thrift:'Limit'(). -type limit_id() :: limproto_limiter_thrift:'LimitID'(). -type limit_version() :: limproto_limiter_thrift:'Version'(). -type limit_change() :: limproto_limiter_thrift:'LimitChange'(). -type context() :: limproto_limiter_thrift:'LimitContext'(). -type clock() :: limproto_limiter_thrift:'Clock'(). +-type request() :: limproto_limiter_thrift:'LimitRequest'(). -export_type([limit/0]). -export_type([limit_id/0]). @@ -21,7 +28,7 @@ -export_type([context/0]). -export_type([clock/0]). --spec get(limit_id(), limit_version(), clock(), context()) -> limit() | no_return(). +-spec get(limit_id(), limit_version() | undefined, clock(), context()) -> limit() | no_return(). get(LimitID, Version, Clock, Context) -> Args = {LimitID, Version, Clock, Context}, Opts = hg_woody_wrapper:get_service_options(limiter), @@ -66,3 +73,49 @@ rollback(LimitChange, Clock, Context) -> {exception, Exception} -> error(Exception) end. + +-spec get_values(request(), context()) -> [limit()] | no_return(). +get_values(Request, Context) -> + {ok, Limits} = call_w_request('GetValues', Request, Context), + Limits. + +-spec get_batch(request(), context()) -> [limit()] | no_return(). +get_batch(Request, Context) -> + {ok, Limits} = call_w_request('GetBatch', Request, Context), + Limits. + +-spec hold_batch(request(), context()) -> [limit()] | no_return(). +hold_batch(Request, Context) -> + {ok, Limits} = call_w_request('HoldBatch', Request, Context), + Limits. + +-spec commit_batch(request(), context()) -> ok | no_return(). +commit_batch(Request, Context) -> + {ok, ok} = call_w_request('CommitBatch', Request, Context), + ok. + +-spec rollback_batch(request(), context()) -> ok | no_return(). +rollback_batch(Request, Context) -> + {ok, ok} = call_w_request('RollbackBatch', Request, Context), + ok. + +%% + +call_w_request(Function, Request, Context) -> + Args = {Request, Context}, + Opts = hg_woody_wrapper:get_service_options(limiter), + case hg_woody_wrapper:call(limiter, Function, Args, Opts) of + {exception, #limiter_LimitNotFound{}} -> + error(not_found); + {exception, #base_InvalidRequest{errors = Errors}} -> + error({invalid_request, Errors}); + {exception, Exception} -> + %% NOTE Uniform handling of more specific exceptions: + %% LimitChangeNotFound + %% InvalidOperationCurrency + %% OperationContextNotSupported + %% PaymentToolNotSupported + error(Exception); + {ok, _} = Result -> + Result + end. diff --git a/apps/hellgate/test/hg_dummy_limiter.erl b/apps/hellgate/test/hg_dummy_limiter.erl deleted file mode 100644 index 811b9731..00000000 --- a/apps/hellgate/test/hg_dummy_limiter.erl +++ /dev/null @@ -1,52 +0,0 @@ --module(hg_dummy_limiter). - --include_lib("limiter_proto/include/limproto_limiter_thrift.hrl"). - --export([new/0]). --export([get/4]). --export([hold/3]). --export([commit/3]). - --type client() :: woody_context:ctx(). - --type limit_id() :: limproto_limiter_thrift:'LimitID'(). --type limit_version() :: limproto_limiter_thrift:'Version'(). --type limit_change() :: limproto_limiter_thrift:'LimitChange'(). --type limit_context() :: limproto_limiter_thrift:'LimitContext'(). --type clock() :: limproto_limiter_thrift:'Clock'(). - -%%% API - --spec new() -> client(). -new() -> - woody_context:new(). - --spec get(limit_id(), limit_version(), limit_context(), client()) -> woody:result() | no_return(). -get(LimitID, Version, Context, Client) -> - call('GetVersioned', {LimitID, Version, clock(), Context}, Client). - --spec hold(limit_change(), limit_context(), client()) -> woody:result() | no_return(). -hold(LimitChange, Context, Client) -> - call('Hold', {LimitChange, clock(), Context}, Client). - --spec commit(limit_change(), limit_context(), client()) -> woody:result() | no_return(). -commit(LimitChange, Context, Client) -> - call('Commit', {LimitChange, clock(), Context}, Client). - -%%% Internal functions - --spec call(atom(), tuple(), client()) -> woody:result() | no_return(). -call(Function, Args, Client) -> - Call = {{limproto_limiter_thrift, 'Limiter'}, Function, Args}, - Opts = #{ - url => <<"http://limiter:8022/v1/limiter">>, - event_handler => scoper_woody_event_handler, - transport_opts => #{ - max_connections => 10000 - } - }, - woody_client:call(Call, Opts, Client). - --spec clock() -> clock(). -clock() -> - {vector, #limiter_VectorClock{state = <<>>}}. diff --git a/apps/hellgate/test/hg_invoice_tests_SUITE.erl b/apps/hellgate/test/hg_invoice_tests_SUITE.erl index 33ce7fe0..58474f6f 100644 --- a/apps/hellgate/test/hg_invoice_tests_SUITE.erl +++ b/apps/hellgate/test/hg_invoice_tests_SUITE.erl @@ -527,8 +527,8 @@ init_per_suite(C) -> {cowboy, CowboySpec} ]), - _ = hg_limiter_helper:init_per_suite(C), - _ = hg_domain:insert(construct_domain_fixture()), + BaseLimitsRevision = hg_limiter_helper:init_per_suite(C), + _BaseRevision = hg_domain:insert(construct_domain_fixture()), RootUrl = maps:get(hellgate_root_url, Ret), @@ -562,7 +562,8 @@ init_per_suite(C) -> {another_customer_client, CustomerClient2}, {root_url, RootUrl}, {apps, Apps}, - {test_sup, SupPid} + {test_sup, SupPid}, + {base_limits_domain_revision, BaseLimitsRevision} | C ], @@ -717,7 +718,7 @@ init_per_testcase(Name = repair_fail_routing_succeeded, C) -> meck:expect( hg_limiter, check_limits, - fun override_check_limits/4 + fun override_check_limits/5 ), init_per_testcase_(Name, C); init_per_testcase(Name = repair_fail_cash_flow_building_succeeded, C) -> @@ -738,15 +739,15 @@ init_per_testcase(Name, C) -> end, init_per_testcase_(Name, C1). -override_check_limits(_, _, _, _) -> throw(unknown). --dialyzer({nowarn_function, override_check_limits/4}). +override_check_limits(_, _, _, _, _) -> throw(unknown). +-dialyzer({nowarn_function, override_check_limits/5}). override_collect_cashflow(_) -> throw(unknown). -dialyzer({nowarn_function, override_collect_cashflow/1}). override_domain_fixture(Fixture, C) -> Revision = hg_domain:head(), - _ = hg_domain:upsert(Fixture(Revision, C)), + _NewRevision = hg_domain:upsert(Fixture(Revision, C)), [{original_domain_revision, Revision} | C]. override_domain_fixture(Fixture, Name, C) -> @@ -1343,7 +1344,9 @@ payment_limit_overflow(C) -> ) = create_payment(PartyID, ShopID, PaymentAmount, Client, PmtSys), Failure = create_payment_limit_overflow(PartyID, ShopID, 1000, Client, PmtSys), - ok = hg_limiter_helper:assert_payment_limit_amount(PaymentAmount, Payment, Invoice), + ok = hg_limiter_helper:assert_payment_limit_amount( + ?LIMIT_ID, configured_limit_version(?LIMIT_ID, C), PaymentAmount, Payment, Invoice + ), ok = payproc_errors:match('PaymentFailure', Failure, fun({no_route_found, {rejected, {limit_overflow, _}}}) -> ok end). @@ -1409,7 +1412,9 @@ switch_provider_after_limit_overflow(C) -> [?payment_state(Payment)] ) = create_payment(PartyID, ShopID, PaymentAmount, Client, PmtSys), - ok = hg_limiter_helper:assert_payment_limit_amount(PaymentAmount, Payment, Invoice), + ok = hg_limiter_helper:assert_payment_limit_amount( + ?LIMIT_ID, configured_limit_version(?LIMIT_ID, C), PaymentAmount, Payment, Invoice + ), #domain_InvoicePayment{id = PaymentID} = Payment, InvoiceID = start_invoice(PartyID, ShopID, <<"rubberduck">>, make_due_date(10), PaymentAmount, Client), @@ -6710,7 +6715,11 @@ payment_cascade_success(C) -> }, #payproc_InvoicePayment{payment = Payment} = hg_client_invoicing:start_payment(InvoiceID, PaymentParams, Client), ?invoice_created(?invoice_w_status(?invoice_unpaid())) = next_change(InvoiceID, Client), - {ok, Limit} = hg_limiter_helper:get_payment_limit_amount(?LIMIT_ID4, hg_domain:head(), Payment, Invoice), + Limit = hg_limiter_helper:maybe_uninitialized_limit( + hg_limiter_helper:get_payment_limit_amount( + ?LIMIT_ID4, configured_limit_version(?LIMIT_ID4, C), Payment, Invoice + ) + ), InitialAccountedAmount = hg_limiter_helper:get_amount(Limit), [ ?payment_ev(PaymentID, ?payment_started(?payment_w_status(?pending()))), @@ -6761,7 +6770,7 @@ payment_cascade_success(C) -> ), %% At the end of this scenario limit must be accounted only once. _ = hg_limiter_helper:assert_payment_limit_amount( - ?LIMIT_ID4, InitialAccountedAmount + Amount, PaymentFinal, Invoice + ?LIMIT_ID4, configured_limit_version(?LIMIT_ID4, C), InitialAccountedAmount + Amount, PaymentFinal, Invoice ), #payproc_InvoicePaymentExplanation{ explained_routes = [ @@ -7098,7 +7107,11 @@ payment_cascade_limit_overflow(C) -> }, #payproc_InvoicePayment{payment = Payment} = hg_client_invoicing:start_payment(InvoiceID, PaymentParams, Client), ?invoice_created(?invoice_w_status(?invoice_unpaid())) = next_change(InvoiceID, Client), - {ok, Limit} = hg_limiter_helper:get_payment_limit_amount(?LIMIT_ID4, hg_domain:head(), Payment, Invoice), + Limit = hg_limiter_helper:maybe_uninitialized_limit( + hg_limiter_helper:get_payment_limit_amount( + ?LIMIT_ID4, configured_limit_version(?LIMIT_ID4, C), Payment, Invoice + ) + ), InitialAccountedAmount = hg_limiter_helper:get_amount(Limit), [ ?payment_ev(PaymentID, ?payment_started(?payment_w_status(?pending()))), @@ -7126,7 +7139,9 @@ payment_cascade_limit_overflow(C) -> ?assertMatch(#domain_InvoicePayment{status = {failed, _}}, FinalPayment), ?invoice_status_changed(?invoice_cancelled(<<"overdue">>)) = next_change(InvoiceID, Client), %% At the end of this scenario limit must not be changed. - hg_limiter_helper:assert_payment_limit_amount(?LIMIT_ID4, InitialAccountedAmount, FinalPayment, Invoice). + hg_limiter_helper:assert_payment_limit_amount( + ?LIMIT_ID4, configured_limit_version(?LIMIT_ID4, C), InitialAccountedAmount, FinalPayment, Invoice + ). -spec payment_big_cascade_success(config()) -> test_return(). payment_big_cascade_success(C) -> @@ -7154,7 +7169,11 @@ payment_big_cascade_success(C) -> }, #payproc_InvoicePayment{payment = Payment} = hg_client_invoicing:start_payment(InvoiceID, PaymentParams, Client), ?invoice_created(?invoice_w_status(?invoice_unpaid())) = next_change(InvoiceID, Client), - {ok, Limit} = hg_limiter_helper:get_payment_limit_amount(?LIMIT_ID4, hg_domain:head(), Payment, Invoice), + Limit = hg_limiter_helper:maybe_uninitialized_limit( + hg_limiter_helper:get_payment_limit_amount( + ?LIMIT_ID4, configured_limit_version(?LIMIT_ID4, C), Payment, Invoice + ) + ), InitialAccountedAmount = hg_limiter_helper:get_amount(Limit), [ ?payment_ev(PaymentID, ?payment_started(?payment_w_status(?pending()))), @@ -7210,7 +7229,9 @@ payment_big_cascade_success(C) -> Trx ), %% At the end of this scenario limit must be accounted only once. - hg_limiter_helper:assert_payment_limit_amount(?LIMIT_ID4, InitialAccountedAmount + Amount, PaymentFinal, Invoice). + hg_limiter_helper:assert_payment_limit_amount( + ?LIMIT_ID4, configured_limit_version(?LIMIT_ID4, C), InitialAccountedAmount + Amount, PaymentFinal, Invoice + ). payment_cascade_fail_provider_error_fixture_pre(Revision, _C) -> lists:flatten([ @@ -10823,3 +10844,6 @@ mock_fault_detector(SupPid) -> ], SupPid ). + +configured_limit_version(_LimitID, C) -> + genlib:define(cfg(original_domain_revision, C), cfg(base_limits_domain_revision, C)). diff --git a/apps/hellgate/test/hg_limiter_helper.erl b/apps/hellgate/test/hg_limiter_helper.erl index 4730e414..79f141e1 100644 --- a/apps/hellgate/test/hg_limiter_helper.erl +++ b/apps/hellgate/test/hg_limiter_helper.erl @@ -2,7 +2,6 @@ -include_lib("limiter_proto/include/limproto_limiter_thrift.hrl"). -include_lib("limiter_proto/include/limproto_context_payproc_thrift.hrl"). --include_lib("damsel/include/dmsl_domain_conf_thrift.hrl"). -include_lib("damsel/include/dmsl_domain_thrift.hrl"). -include_lib("damsel/include/dmsl_limiter_config_thrift.hrl"). @@ -10,8 +9,8 @@ -export([init_per_suite/1]). -export([get_amount/1]). --export([assert_payment_limit_amount/3]). --export([assert_payment_limit_amount/4]). +-export([assert_payment_limit_amount/5]). +-export([maybe_uninitialized_limit/1]). -export([get_payment_limit_amount/4]). -export([mk_config_object/2, mk_config_object/3, mk_config_object/4]). -export([mk_context_type/1]). @@ -25,30 +24,39 @@ -define(LIMIT_ID4, <<"ID4">>). -define(SHOPLIMIT_ID, <<"SHOPLIMITID">>). --spec init_per_suite(config()) -> _. +-define(PLACEHOLDER_UNINITIALIZED_LIMIT_ID, <<"uninitialized limit">>). +-define(PLACEHOLDER_OPERATION_GET_LIMIT_VALUES, <<"get values">>). + +-spec init_per_suite(config()) -> dmt_client:vsn(). init_per_suite(_Config) -> - _ = dmt_client:upsert({limit_config, mk_config_object(?LIMIT_ID)}), - _ = dmt_client:upsert({limit_config, mk_config_object(?LIMIT_ID2)}), - _ = dmt_client:upsert({limit_config, mk_config_object(?LIMIT_ID3)}), - _ = dmt_client:upsert({limit_config, mk_config_object(?LIMIT_ID4)}), - _ = dmt_client:upsert({limit_config, mk_config_object(?SHOPLIMIT_ID)}). + dmt_client:upsert([ + {limit_config, mk_config_object(?LIMIT_ID)}, + {limit_config, mk_config_object(?LIMIT_ID2)}, + {limit_config, mk_config_object(?LIMIT_ID3)}, + {limit_config, mk_config_object(?LIMIT_ID4)}, + {limit_config, mk_config_object(?SHOPLIMIT_ID)} + ]). -spec get_amount(_) -> pos_integer(). get_amount(#limiter_Limit{amount = Amount}) -> Amount. --spec assert_payment_limit_amount(_, _, _) -> _. -assert_payment_limit_amount(AssertAmount, Payment, Invoice) -> - assert_payment_limit_amount(?LIMIT_ID, AssertAmount, Payment, Invoice). - --spec assert_payment_limit_amount(_, _, _, _) -> _. -assert_payment_limit_amount(LimitID, AssertAmount, Payment, Invoice) -> - L = - dmt_client:checkout_versioned_object({'limit_config', #domain_LimitConfigRef{id = LimitID}}), - #domain_conf_VersionedObject{version = Version} = L, - {ok, Limit} = get_payment_limit_amount(LimitID, Version, Payment, Invoice), +-spec assert_payment_limit_amount(_, _, _, _, _) -> _. +assert_payment_limit_amount(LimitID, Version, AssertAmount, Payment, Invoice) -> + Limit = maybe_uninitialized_limit(get_payment_limit_amount(LimitID, Version, Payment, Invoice)), ?assertMatch(#limiter_Limit{amount = AssertAmount}, Limit). +-spec maybe_uninitialized_limit({ok, _} | {exception, _}) -> _Limit. +maybe_uninitialized_limit({ok, Limit}) -> + Limit; +maybe_uninitialized_limit({exception, _}) -> + #limiter_Limit{ + id = ?PLACEHOLDER_UNINITIALIZED_LIMIT_ID, + amount = 0, + creation_time = undefined, + description = undefined + }. + -spec get_payment_limit_amount(_, _, _, _) -> _. get_payment_limit_amount(LimitId, Version, Payment, Invoice) -> Context = #limiter_LimitContext{ @@ -62,7 +70,19 @@ get_payment_limit_amount(LimitId, Version, Payment, Invoice) -> } } }, - hg_dummy_limiter:get(LimitId, Version, Context, hg_dummy_limiter:new()). + LimitRequest = #limiter_LimitRequest{ + operation_id = ?PLACEHOLDER_OPERATION_GET_LIMIT_VALUES, + limit_changes = [#limiter_LimitChange{id = LimitId, version = Version}] + }, + try hg_limiter_client:get_values(LimitRequest, Context) of + [L] -> + {ok, L}; + _ -> + {exception, #limiter_LimitNotFound{}} + catch + error:not_found -> + {exception, #limiter_LimitNotFound{}} + end. mk_config_object(LimitID) -> mk_config_object(LimitID, <<"RUB">>). diff --git a/compose.yaml b/compose.yaml index ebb7c109..c407ce86 100644 --- a/compose.yaml +++ b/compose.yaml @@ -63,13 +63,15 @@ services: retries: 10 limiter: - image: ghcr.io/valitydev/limiter:sha-920d6ac + image: ghcr.io/valitydev/limiter:sha-2271094 command: /opt/limiter/bin/limiter foreground depends_on: machinegun: condition: service_healthy shumway: condition: service_started + liminator: + condition: service_healthy healthcheck: test: "/opt/limiter/bin/limiter ping" interval: 5s @@ -96,6 +98,36 @@ services: healthcheck: disable: true + liminator: + image: ghcr.io/valitydev/liminator:sha-fc6546f + restart: unless-stopped + entrypoint: + - java + - -Xmx512m + - -jar + - /opt/liminator/liminator.jar + - --spring.datasource.url=jdbc:postgresql://liminator-db:5432/liminator + - --spring.datasource.username=vality + - --spring.datasource.password=postgres + - --spring.flyway.url=jdbc:postgresql://liminator-db:5432/liminator + - --spring.flyway.username=vality + - --spring.flyway.password=postgres + - --service.skipExistedHoldOps=false + depends_on: + - liminator-db + healthcheck: + test: "curl http://localhost:8022/actuator/health" + interval: 5s + timeout: 1s + retries: 20 + + liminator-db: + image: docker.io/library/postgres:13.10 + environment: + - POSTGRES_DB=liminator + - POSTGRES_USER=vality + - POSTGRES_PASSWORD=postgres + party-management: image: ghcr.io/valitydev/party-management:sha-b78d0f5 command: /opt/party-management/bin/party-management foreground @@ -114,8 +146,6 @@ services: shumway-db: image: docker.io/library/postgres:13.10 - ports: - - "5432" environment: - POSTGRES_DB=shumway - POSTGRES_USER=postgres diff --git a/rebar.lock b/rebar.lock index e5534513..28321322 100644 --- a/rebar.lock +++ b/rebar.lock @@ -51,7 +51,7 @@ {<<"jsx">>,{pkg,<<"jsx">>,<<"3.1.0">>},1}, {<<"limiter_proto">>, {git,"https://github.com/valitydev/limiter-proto.git", - {ref,"10328404f1cea68586962ed7fce0405b18d62b28"}}, + {ref,"970f197ce6c527fee5c45237ad2ce4b8820184a1"}}, 0}, {<<"metrics">>,{pkg,<<"metrics">>,<<"1.0.1">>},2}, {<<"mg_proto">>,