From 6f0bc740273906b1872654bb32791cfc9123c21e Mon Sep 17 00:00:00 2001 From: "Andrey N. Ronin" Date: Sun, 24 Apr 2022 20:06:54 +0400 Subject: [PATCH] Introduce Steps * Add exponent retry * Dynamically start Finch pool with the given proxy configs * Support multiple content-encoding headers * Add follow_redirects step --- .github/workflows/main.yml | 48 +- lib/http_client/adapter.ex | 35 +- lib/http_client/adapters/finch.ex | 139 ++--- lib/http_client/adapters/finch/config.ex | 44 -- lib/http_client/adapters/httpoison.ex | 74 +-- lib/http_client/application.ex | 5 +- lib/http_client/error.ex | 14 +- lib/http_client/request.ex | 245 ++++++++ lib/http_client/response.ex | 38 +- lib/http_client/steps.ex | 678 +++++++++++++++++++++++ lib/http_client/telemetry.ex | 6 +- mix.exs | 5 +- mix.lock | 8 +- test/http_client_test.exs | 48 +- 14 files changed, 1123 insertions(+), 264 deletions(-) delete mode 100644 lib/http_client/adapters/finch/config.ex create mode 100644 lib/http_client/request.ex create mode 100644 lib/http_client/steps.ex diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5d42eb0..5b07465 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,49 +3,35 @@ name: CI on: [push, pull_request] jobs: - format: - name: Format and compile with warnings as errors - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install OTP and Elixir - uses: erlef/setup-beam@v1 - with: - otp-version: 27.x - elixir-version: 1.18.x - - - name: Install dependencies - run: mix deps.get - - - name: Run "mix format" - run: mix format --check-formatted - - - name: Compile with --warnings-as-errors - run: mix compile --warnings-as-errors - test: - name: Test runs-on: ubuntu-latest + env: + MIX_ENV: test strategy: fail-fast: false matrix: include: - - erlang: 26.x - elixir: 1.16.x - - erlang: 27.x - elixir: 1.19.x + - elixir: 1.16.x + otp: 26.x + - elixir: 1.19.x + otp: 27.x steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install OTP and Elixir - uses: erlef/setup-elixir@v1 + uses: erlef/setup-beam@v1 with: - otp-version: ${{matrix.erlang}} + otp-version: ${{matrix.otp}} elixir-version: ${{matrix.elixir}} - name: Install dependencies run: mix deps.get - - name: Run tests - run: mix test --trace + - name: Run "mix format" + run: mix format --check-formatted + + - name: Check unused dependencies + run: mix deps.unlock --check-unused + + - name: Compile with --warnings-as-errors + run: mix compile --warnings-as-errors diff --git a/lib/http_client/adapter.ex b/lib/http_client/adapter.ex index 7300577..e1955c5 100644 --- a/lib/http_client/adapter.ex +++ b/lib/http_client/adapter.ex @@ -16,7 +16,7 @@ defmodule HTTPClient.Adapter do """ alias HTTPClient.Adapters.{Finch, HTTPoison} - alias HTTPClient.{Error, Response, Telemetry} + alias HTTPClient.{Request, Steps} alias NimbleOptions.ValidationError @typedoc """ @@ -123,50 +123,39 @@ defmodule HTTPClient.Adapter do @doc false def request(adapter, method, url, body, headers, options) do - perform(adapter, :request, [method, url, body, headers, options]) + perform(adapter, method, url, body: body, headers: headers, options: options) end @doc false def get(adapter, url, headers, options) do - perform(adapter, :get, [url, headers, options]) + perform(adapter, :get, url, headers: headers, options: options) end @doc false def post(adapter, url, body, headers, options) do - perform(adapter, :post, [url, body, headers, options]) + perform(adapter, :post, url, body: body, headers: headers, options: options) end @doc false def put(adapter, url, body, headers, options) do - perform(adapter, :put, [url, body, headers, options]) + perform(adapter, :put, url, body: body, headers: headers, options: options) end @doc false def patch(adapter, url, body, headers, options) do - perform(adapter, :patch, [url, body, headers, options]) + perform(adapter, :patch, url, body: body, headers: headers, options: options) end @doc false def delete(adapter, url, headers, options) do - perform(adapter, :delete, [url, headers, options]) + perform(adapter, :delete, url, headers: headers, options: options) end - defp perform(adapter, method, args) do - metadata = %{adapter: adapter, args: args, method: method} - start_time = Telemetry.start(:request, metadata) - - case apply(adapter, method, args) do - {:ok, %Response{status: status, headers: headers} = response} -> - metadata = Map.put(metadata, :status_code, status) - Telemetry.stop(:request, start_time, metadata) - headers = Enum.map(headers, fn {key, value} -> {String.downcase(key), value} end) - {:ok, %{response | headers: headers}} - - {:error, %Error{reason: reason}} = error_response -> - metadata = Map.put(metadata, :error, reason) - Telemetry.stop(:request, start_time, metadata) - error_response - end + defp perform(adapter, method, url, options) do + adapter + |> Request.build(method, url, options) + |> Steps.put_default_steps() + |> Request.run() end defp adapter_mod(:finch), do: HTTPClient.Adapters.Finch diff --git a/lib/http_client/adapters/finch.ex b/lib/http_client/adapters/finch.ex index 81de692..3ce81db 100644 --- a/lib/http_client/adapters/finch.ex +++ b/lib/http_client/adapters/finch.ex @@ -3,7 +3,7 @@ defmodule HTTPClient.Adapters.Finch do Implementation of `HTTPClient.Adapter` behaviour using Finch HTTP client. """ - alias HTTPClient.{Error, Response} + alias HTTPClient.{Request, Response} @type method() :: Finch.Request.method() @type url() :: Finch.Request.url() @@ -11,110 +11,89 @@ defmodule HTTPClient.Adapters.Finch do @type body() :: Finch.Request.body() @type options() :: keyword() - @behaviour HTTPClient.Adapter + @doc """ + Performs the request using `Finch`. + """ + def perform_request(request) do + options = prepare_options(request.options) - @delay 1000 + request.method + |> Finch.build(request.url, request.headers, request.body) + |> Finch.request(request.private.finch_name, options) + |> case do + {:ok, %{status: status, body: body, headers: headers}} -> + {request, + Response.new(status: status, body: body, headers: headers, request_url: request.url)} - @impl true - def request(method, url, body, headers, options) do - perform_request(method, url, headers, body, options) + {:error, exception} -> + {request, exception} + end end - @impl true - def get(url, headers, options) do - perform_request(:get, url, headers, nil, options) + @doc false + def proxy(request) do + Request.put_private(request, :finch_name, get_client()) end - @impl true - def post(url, body, headers, options) do - perform_request(:post, url, headers, body, options) + defp prepare_options(options) do + Enum.map(options, &normalize_option/1) end - @impl true - def put(url, body, headers, options) do - perform_request(:put, url, headers, body, options) - end + defp normalize_option({:timeout, value}), do: {:pool_timeout, value} + defp normalize_option({:recv_timeout, value}), do: {:receive_timeout, value} + defp normalize_option({key, value}), do: {key, value} - @impl true - def patch(url, body, headers, options) do - perform_request(:patch, url, headers, body, options) + defp get_client() do + :http_client + |> Application.get_env(:proxy) + |> get_client_name() end - @impl true - def delete(url, headers, options) do - perform_request(:delete, url, headers, nil, options) + defp get_client_name(nil), do: HTTPClient.Finch + + defp get_client_name(proxies) when is_list(proxies) do + proxies + |> Enum.random() + |> get_client_name() end - defp perform_request(method, url, headers, body, options, attempt \\ 0) do - {params, options} = Keyword.pop(options, :params) - {basic_auth, options} = Keyword.pop(options, :basic_auth) + defp get_client_name(proxy) when is_map(proxy) do + name = custom_pool_name(proxy) - url = build_request_url(url, params) - headers = add_basic_auth_header(headers, basic_auth) - options = prepare_options(options) + pools = %{ + default: [ + conn_opts: [proxy: compose_proxy(proxy), proxy_headers: compose_proxy_headers(proxy)] + ] + } - method - |> Finch.build(url, headers, body) - |> Finch.request(get_client(), options) - |> case do - {:ok, %{status: status, body: body, headers: headers}} -> - {:ok, %Response{status: status, body: body, headers: headers, request_url: url}} - - {:error, - %Mint.HTTPError{ - reason: {:proxy, _} - }} -> - case attempt < 5 do - true -> - Process.sleep(attempt * @delay) - perform_request(method, url, headers, body, options, attempt + 1) - - false -> - {:error, %Error{reason: :proxy_error}} - end - - {:error, error} -> - {:error, %Error{reason: error.reason}} - end - end - - defp build_request_url(url, nil), do: url + child_spec = {Finch, name: name, pools: pools} - defp build_request_url(url, params) do - cond do - Enum.count(params) === 0 -> url - URI.parse(url).query -> url <> "&" <> URI.encode_query(params) - true -> url <> "?" <> URI.encode_query(params) + case DynamicSupervisor.start_child(HTTPClient.FinchSupervisor, child_spec) do + {:ok, _} -> name + {:error, {:already_started, _}} -> name end end - defp add_basic_auth_header(headers, {username, password}) do - credentials = Base.encode64("#{username}:#{password}") - [{"Authorization", "Basic " <> credentials} | headers || []] + defp compose_proxy_headers(%{opts: opts}) do + Keyword.get(opts, :proxy_headers, []) end - defp add_basic_auth_header(headers, _basic_auth), do: headers + defp compose_proxy_headers(_), do: [] - defp prepare_options(options) do - Enum.map(options, &normalize_option/1) + defp compose_proxy(proxy) do + {proxy.scheme, proxy.address, to_integer(proxy.port), proxy.opts} end - defp normalize_option({:timeout, value}), do: {:pool_timeout, value} - defp normalize_option({:recv_timeout, value}), do: {:receive_timeout, value} - defp normalize_option({key, value}), do: {key, value} - - defp get_client do - case Application.get_env(:http_client, :proxy, nil) do - nil -> FinchHTTPClient - proxies -> get_client_with_proxy(proxies) - end - end + defp to_integer(term) when is_integer(term), do: term + defp to_integer(term) when is_binary(term), do: String.to_integer(term) - defp get_client_with_proxy(proxy) when is_map(proxy) do - FinchHTTPClientWithProxy_0 - end + defp custom_pool_name(opts) do + name = + opts + |> :erlang.term_to_binary() + |> :erlang.md5() + |> Base.url_encode64(padding: false) - defp get_client_with_proxy(proxies) when is_list(proxies) do - :"FinchHTTPClientWithProxy_#{Enum.random(0..(length(proxies) - 1))}" + Module.concat(HTTPClient.FinchSupervisor, "Pool_#{name}") end end diff --git a/lib/http_client/adapters/finch/config.ex b/lib/http_client/adapters/finch/config.ex deleted file mode 100644 index 307d150..0000000 --- a/lib/http_client/adapters/finch/config.ex +++ /dev/null @@ -1,44 +0,0 @@ -defmodule HTTPClient.Adapters.Finch.Config do - @moduledoc """ - Provide Application.children for Application supervisor - """ - - @doc """ - Returns list of childrens for Application supervisor - """ - def children do - case Application.get_env(:http_client, :proxy, nil) do - nil -> [{Finch, name: FinchHTTPClient}] - proxies -> generate_finch_proxies(proxies) - end - end - - defp generate_finch_proxies(proxy) when is_map(proxy) do - [ - { - Finch, - name: FinchHTTPClientWithProxy_0, - pools: %{ - default: [conn_opts: [proxy: {proxy.scheme, proxy.address, proxy.port, proxy.opts}]] - } - } - ] - end - - defp generate_finch_proxies(proxies) when is_list(proxies) do - proxies - |> Enum.with_index() - |> Enum.map(fn {proxy, index} -> - Supervisor.child_spec( - { - Finch, - name: :"FinchHTTPClientWithProxy_#{index}", - pools: %{ - default: [conn_opts: [proxy: {proxy.scheme, proxy.address, proxy.port, proxy.opts}]] - } - }, - id: :"FinchHTTPClientWithProxy_#{index}" - ) - end) - end -end diff --git a/lib/http_client/adapters/httpoison.ex b/lib/http_client/adapters/httpoison.ex index 73d3578..08f1718 100644 --- a/lib/http_client/adapters/httpoison.ex +++ b/lib/http_client/adapters/httpoison.ex @@ -3,7 +3,7 @@ defmodule HTTPClient.Adapters.HTTPoison do Implementation of `HTTPClient.Adapter` behaviour using HTTPoison HTTP client. """ - alias HTTPClient.{Error, Response} + alias HTTPClient.Response @type method() :: HTTPoison.Request.method() @type url() :: HTTPoison.Request.url() @@ -11,67 +11,25 @@ defmodule HTTPClient.Adapters.HTTPoison do @type body() :: HTTPoison.Request.body() @type options() :: HTTPoison.Request.options() - @behaviour HTTPClient.Adapter - - @delay 1000 - - @impl true - def request(method, url, body, headers, options) do - perform_request(method, url, headers, body, options) - end - - @impl true - def get(url, headers, options) do - perform_request(:get, url, headers, "", options) - end - - @impl true - def post(url, body, headers, options) do - perform_request(:post, url, headers, body, options) - end - - @impl true - def put(url, body, headers, options) do - perform_request(:put, url, headers, body, options) - end - - @impl true - def patch(url, body, headers, options) do - perform_request(:patch, url, headers, body, options) - end - - @impl true - def delete(url, headers, options) do - perform_request(:delete, url, headers, "", options) - end - - defp perform_request(method, url, headers, body, options, attempt \\ 0) do - options = setup_proxy(options) - options = add_basic_auth_option(options, options[:basic_auth]) - - case HTTPoison.request(method, url, body, headers, options) do - {:ok, %{status_code: status, body: body, headers: headers, request: request}} -> - {:ok, %Response{status: status, body: body, headers: headers, request_url: request.url}} - - {:error, %HTTPoison.Error{id: nil, reason: :proxy_error}} -> - case attempt < 5 do - true -> - Process.sleep(attempt * @delay) - perform_request(method, url, headers, body, options, attempt + 1) + @doc """ + Performs the request using `HTTPoison`. + """ + def perform_request(request) do + options = Map.to_list(request.options) - false -> - {:error, %Error{reason: :proxy_error}} - end + case HTTPoison.request(request.method, request.url, request.body, request.headers, options) do + {:ok, %{status_code: status, body: body, headers: headers}} -> + {request, + Response.new(status: status, body: body, headers: headers, request_url: request.url)} - {:error, error} -> - {:error, %Error{reason: error.reason}} + {:error, exception} -> + {request, exception} end end - defp add_basic_auth_option(options, nil), do: options - - defp add_basic_auth_option(options, {username, password}) do - put_in(options, [:hackney], basic_auth: {username, password}) + @doc false + def proxy(request) do + update_in(request.options, &setup_proxy/1) end defp setup_proxy(options) do @@ -82,7 +40,7 @@ defmodule HTTPClient.Adapters.HTTPoison do end defp add_proxy(options, proxy) when is_map(proxy) do - Keyword.put(options, :proxy, "#{proxy.scheme}://#{proxy.address}:#{proxy.port}") + Map.put(options, :proxy, "#{proxy.scheme}://#{proxy.address}:#{proxy.port}") end defp add_proxy(options, proxies) when is_list(proxies) do diff --git a/lib/http_client/application.ex b/lib/http_client/application.ex index f8e1a36..c892df5 100644 --- a/lib/http_client/application.ex +++ b/lib/http_client/application.ex @@ -4,7 +4,10 @@ defmodule HTTPClient.Application do use Application def start(_type, _args) do - children = [] ++ HTTPClient.Adapters.Finch.Config.children() + children = [ + {Finch, name: HTTPClient.Finch}, + {DynamicSupervisor, strategy: :one_for_one, name: HTTPClient.FinchSupervisor} + ] opts = [strategy: :one_for_one, name: HTTPClient.Supervisor] Supervisor.start_link(children, opts) diff --git a/lib/http_client/error.ex b/lib/http_client/error.ex index 272906e..709b808 100644 --- a/lib/http_client/error.ex +++ b/lib/http_client/error.ex @@ -3,9 +3,19 @@ defmodule HTTPClient.Error do An error of a request. """ - defstruct [:reason] + @type t() :: %__MODULE__{reason: atom()} - @type t :: %__MODULE__{reason: term()} + defexception [:reason] + + @impl true + def exception(reason) when is_atom(reason) do + %__MODULE__{reason: reason} + end + + @impl true + def message(%__MODULE__{reason: reason}) do + "#{reason}" + end defimpl Jason.Encoder do def encode(struct, opts) do diff --git a/lib/http_client/request.ex b/lib/http_client/request.ex new file mode 100644 index 0000000..75d9cc0 --- /dev/null +++ b/lib/http_client/request.ex @@ -0,0 +1,245 @@ +defmodule HTTPClient.Request do + @moduledoc """ + The request struct. + + Struct fields: + + * `:adapter` - an implementation of adapter to use + + * `:options` - steps and adapter options + + * `:method` - the HTTP request method + + * `:url` - the HTTP request URL + + * `:headers` - the HTTP request headers + + * `:body` - the HTTP request body + + * `:halted` - whether the request pipeline is halted. See `halt/1` + + * `:request_steps` - the list of request steps + + * `:response_steps` - the list of response steps + + * `:error_steps` - the list of error steps + + * `:private` - a map reserved for internal use. + + """ + + alias HTTPClient.{Error, Request, Response} + + defstruct [ + :adapter, + method: :get, + url: "", + options: [], + headers: [], + body: "", + halted: false, + request_steps: [], + response_steps: [], + error_steps: [], + private: %{} + ] + + @doc """ + Gets the value for a specific private `key`. + """ + def get_private(request, key, default \\ nil) when is_atom(key) do + Map.get(request.private, key, default) + end + + @doc """ + Assigns a private `key` to `value`. + """ + def put_private(request, key, value) when is_atom(key) do + put_in(request.private[key], value) + end + + @doc """ + Halts the request preventing any further steps from executing. + """ + def halt(request) do + %{request | halted: true} + end + + @doc """ + Builds a request. + """ + def build(adapter, method, url, options \\ []) do + %__MODULE__{ + adapter: adapter, + options: prepare_options(options), + method: method, + url: URI.parse(url), + headers: Keyword.get(options, :headers, []), + body: Keyword.get(options, :body, "") + } + end + + @doc """ + Prepends adapter step to request steps. + """ + def prepend_adapter_step(request) do + adapter = request.adapter + prepend_request_step(request, &adapter.perform_request/1) + end + + @doc """ + Prepends request step. + """ + def prepend_request_step(request, step) do + update_in(request.request_steps, &[step | &1]) + end + + @doc """ + Reverses request steps. + """ + def reverse_request_steps(request) do + update_in(request.request_steps, &Enum.reverse/1) + end + + @doc """ + Prepends response step. + """ + def prepend_response_step(request, step) do + update_in(request.response_steps, &[step | &1]) + end + + @doc """ + Reverses response steps. + """ + def reverse_response_steps(request) do + update_in(request.response_steps, &Enum.reverse/1) + end + + @doc """ + Prepends error step. + """ + def prepend_error_step(request, step) do + update_in(request.error_steps, &[step | &1]) + end + + @doc """ + Reverses error steps. + """ + def reverse_error_steps(request) do + update_in(request.error_steps, &Enum.reverse/1) + end + + @doc """ + Runs a request pipeline. + + Returns `{:ok, response}` or `{:error, exception}`. + """ + def run(request) do + run_request(request.request_steps, request) + end + + defp run_request([step | steps], request) do + case run_step(step, request) do + %Request{} = request -> + run_request(steps, request) + + {%Request{halted: true}, response_or_exception} -> + result(response_or_exception) + + {request, %Response{} = response} -> + run_response(request, response) + + {request, exception} when is_exception(exception) -> + run_error(request, exception) + end + end + + defp run_request([], request) do + adapter = request.adapter + + case run_step(&adapter.perform_request/1, request) do + {request, %Response{} = response} -> + run_response(request, response) + + {request, exception} when is_exception(exception) -> + run_error(request, exception) + + other -> + raise "expected adapter to return {request, response} or {request, exception}, " <> + "got: #{inspect(other)}" + end + end + + defp run_response(request, response) do + steps = request.response_steps + + {_request, response_or_exception} = + Enum.reduce_while(steps, {request, response}, fn step, {request, response} -> + case run_step(step, {request, response}) do + {%Request{halted: true} = request, response_or_exception} -> + {:halt, {request, response_or_exception}} + + {request, %Response{} = response} -> + {:cont, {request, response}} + + {request, exception} when is_exception(exception) -> + {:halt, run_error(request, exception)} + end + end) + + result(response_or_exception) + end + + defp run_error(request, exception) do + steps = request.error_steps + + {_request, response_or_exception} = + Enum.reduce_while(steps, {request, exception}, fn step, {request, exception} -> + case run_step(step, {request, exception}) do + {%Request{halted: true} = request, response_or_exception} -> + {:halt, {request, response_or_exception}} + + {request, exception} when is_exception(exception) -> + {:cont, {request, exception}} + + {request, %Response{} = response} -> + {:halt, run_response(request, response)} + end + end) + + result(response_or_exception) + end + + @doc false + def run_step(step, state) + + def run_step({module, function, args}, state) do + apply(module, function, [state | args]) + end + + def run_step({module, options}, state) do + apply(module, :run, [state | [options]]) + end + + def run_step(module, state) when is_atom(module) do + apply(module, :run, [state, []]) + end + + def run_step(func, state) when is_function(func, 1) do + func.(state) + end + + defp result(%Response{} = response) do + {:ok, response} + end + + defp result(exception) when is_exception(exception) do + {:error, %Error{reason: Exception.message(exception)}} + end + + defp prepare_options(options) do + options + |> Keyword.get(:options, []) + |> Map.new() + end +end diff --git a/lib/http_client/response.ex b/lib/http_client/response.ex index 5880eb2..a2eca9e 100644 --- a/lib/http_client/response.ex +++ b/lib/http_client/response.ex @@ -1,14 +1,48 @@ defmodule HTTPClient.Response do @moduledoc """ A response to a request. + + Fields: + + * `:status` - the HTTP status code + + * `:headers` - the HTTP response headers + + * `:request_url` - the URL of request + + * `:body` - the HTTP response body + + * `:private` - a map reserved for internal use. """ - defstruct [:request_url, :status, body: "", headers: []] + defstruct [:request_url, :status, body: "", headers: [], private: %{}] - @type t :: %__MODULE__{ + @type t() :: %__MODULE__{ body: binary(), headers: keyword(), + private: map(), request_url: binary(), status: non_neg_integer() } + + @doc """ + Builds `HTTPClient.Response` struct with provided data. + """ + def new(data) do + struct(%__MODULE__{}, data) + end + + @doc """ + Gets the value for a specific private `key`. + """ + def get_private(response, key, default \\ nil) when is_atom(key) do + Map.get(response.private, key, default) + end + + @doc """ + Assigns a private `key` to `value`. + """ + def put_private(response, key, value) when is_atom(key) do + put_in(response.private[key], value) + end end diff --git a/lib/http_client/steps.ex b/lib/http_client/steps.ex new file mode 100644 index 0000000..9440c1a --- /dev/null +++ b/lib/http_client/steps.ex @@ -0,0 +1,678 @@ +defmodule HTTPClient.Steps do + @moduledoc """ + A collection of built-in steps. + """ + + require Logger + + alias HTTPClient.{Request, Response, Telemetry} + + @doc """ + Adds default steps. + """ + def put_default_steps(request) do + adapter = request.adapter + + request + |> Request.prepend_request_step(&__MODULE__.encode_headers/1) + |> Request.prepend_request_step(&__MODULE__.put_default_headers/1) + |> Request.prepend_request_step(&__MODULE__.encode_body/1) + |> Request.prepend_request_step(&adapter.proxy/1) + |> Request.prepend_request_step(&__MODULE__.auth/1) + |> Request.prepend_request_step(&__MODULE__.put_params/1) + |> Request.prepend_request_step(&__MODULE__.log_request_start/1) + |> Request.prepend_adapter_step() + |> Request.prepend_response_step(&__MODULE__.downcase_headers/1) + |> Request.prepend_response_step(&__MODULE__.follow_redirects/1) + |> Request.prepend_response_step(&__MODULE__.decompress_body/1) + |> Request.prepend_response_step(&__MODULE__.decode_body/1) + |> Request.prepend_response_step(&__MODULE__.retry/1) + |> Request.prepend_response_step(&__MODULE__.log_response_end/1) + |> Request.prepend_error_step(&__MODULE__.retry/1) + |> Request.reverse_request_steps() + |> Request.reverse_response_steps() + |> Request.reverse_error_steps() + end + + @doc """ + Adds common request headers. + + Currently the following headers are added: + + * `"accept-encoding"` - `"gzip"` + + """ + def put_default_headers(request) do + put_new_header(request, "accept-encoding", "gzip") + end + + @doc """ + Sets request authentication. + + * `:auth` - sets the `authorization` header: + + * `string` - sets to this value; + + * `{:basic, tuple}` - uses Basic HTTP authentication; + + * `{:bearer, token}` - uses Bearer HTTP authentication; + + """ + def auth(request) do + auth(request, Map.get(request.options, :auth)) + end + + defp auth(request, nil), do: request + + defp auth(request, authorization) when is_binary(authorization) do + put_new_header(request, "authorization", authorization) + end + + defp auth(request, {:bearer, token}) when is_binary(token) do + put_new_header(request, "authorization", "Bearer #{token}") + end + + defp auth(request, {:basic, data}) when is_tuple(data) do + 0 + |> Range.new(tuple_size(data) - 1) + |> Enum.map_join(":", &"#{elem(data, &1)}") + |> Base.encode64() + |> then(&put_new_header(request, "authorization", "Basic #{&1}")) + end + + @doc """ + Encodes request headers. + + Turns atom header names into strings, replacing `-` with `_`. For example, `:user_agent` becomes + `"user-agent"`. Non-atom header names are kept as is. + + If a header value is a `NaiveDateTime` or `DateTime`, it is encoded as "HTTP date". Otherwise, + the header value is encoded with `String.Chars.to_string/1`. + """ + def encode_headers(request) do + headers = + for {name, value} <- request.headers do + {prepare_header_name(name), prepare_header_value(value)} + end + + %{request | headers: headers} + end + + @doc """ + Encodes the request body. + + ## Request Options + + * `:form` - if set, encodes the request body as form data (using `URI.encode_query/1`). + + * `:json` - if set, encodes the request body as JSON (using `Jason.encode_to_iodata!/1`), sets + the `accept` header to `application/json`, and the `content-type` + header to `application/json`. + + """ + def encode_body(%{body: {:form, data}} = request) do + request + |> Map.put(:body, URI.encode_query(data)) + |> put_new_header("content-type", "application/x-www-form-urlencoded") + end + + def encode_body(%{body: {:json, data}} = request) do + request + |> Map.put(:body, Jason.encode_to_iodata!(data)) + |> put_new_header("content-type", "application/json") + |> put_new_header("accept", "application/json") + end + + def encode_body(request), do: request + + @doc """ + Adds params to request query string. + """ + def put_params(request) do + put_params(request, get_options(request.options, :params)) + end + + defp put_params(request, []) do + request + end + + defp put_params(request, params) do + encoded = URI.encode_query(params) + + update_in(request.url.query, fn + nil -> encoded + query -> query <> "&" <> encoded + end) + end + + @doc false + def log_request_start(request) do + metadata = %{ + adapter: request.adapter, + headers: request.headers, + method: request.method, + url: to_string(request.url) + } + + start_time = Telemetry.start(:request, metadata) + update_in(request.private, &Map.put(&1, :request_start_time, start_time)) + end + + @doc """ + Decodes response body based on the detected format. + + Supported formats: + + | Format | Decoder | + | ------ | ---------------------------------------------------------------- | + | json | `Jason.decode!/1` | + | gzip | `:zlib.gunzip/1` | + + """ + + def decode_body({request, %{body: ""} = response}), do: {request, response} + + def decode_body({request, response}) when request.options.raw == true do + {request, response} + end + + def decode_body({request, response}) when request.options.decode_body == false do + {request, response} + end + + def decode_body({request, response}) do + case format(request, response) do + "json" -> + {request, update_in(response.body, &Jason.decode!/1)} + + "gz" -> + {request, update_in(response.body, &:zlib.gunzip/1)} + + _ -> + {request, response} + end + end + + defp format(_request, response) do + with {_, content_type} <- List.keyfind(response.headers, "content-type", 0) do + case MIME.extensions(content_type) do + [ext | _] -> ext + [] -> nil + end + end + end + + @doc """ + Follows redirects. + + The original request method may be changed to GET depending on the status code: + + | Code | Method handling | + | ------------- | ------------------ | + | 301, 302, 303 | Changed to GET | + | 307, 308 | Method not changed | + + ## Request Options + + * `:follow_redirects` - if set to `false`, disables automatic response redirects. + Defaults to `true`. + + * `:location_trusted` - by default, authorization credentials are only sent + on redirects with the same host, scheme and port. If `:location_trusted` is set + to `true`, credentials will be sent to any host. + + * `:max_redirects` - the maximum number of redirects, defaults to `10`. + If the limit is reached, an error is raised. + + * `:redirect_log_level` - the log level to emit redirect logs at. Can also be set + to `false` to disable logging these messsages. Defaults to `:debug`. + + """ + def follow_redirects(request_response) + + def follow_redirects({request, response}) when request.options.follow_redirects == false do + {request, response} + end + + def follow_redirects({request, %{status: status} = response}) + when status in [301, 302, 303, 307, 308] do + max_redirects = Map.get(request.options, :max_redirects, 10) + redirect_count = Request.get_private(request, :req_redirect_count, 0) + + if redirect_count < max_redirects do + request = + request + |> build_redirect_request(response) + |> Request.put_private(:req_redirect_count, redirect_count + 1) + + {_, result} = Request.run(request) + {Request.halt(request), result} + else + raise "too many redirects (#{max_redirects})" + end + end + + def follow_redirects(other) do + other + end + + defp build_redirect_request(request, response) do + location = get_header(response.headers, "location") + log_level = Map.get(request.options, :redirect_log_level, :debug) + log_redirect(log_level, location) + location_trusted = Map.get(request.options, :location_trusted) + location_url = URI.merge(request.url, URI.parse(location)) + + request + |> remove_params() + |> remove_credentials_if_untrusted(location_trusted, location_url) + |> put_redirect_request_method(response.status) + |> put_redirect_location(location_url) + end + + defp log_redirect(false, _location), do: :ok + + defp log_redirect(level, location) do + Logger.log(level, ["follow_redirects: redirecting to ", location]) + end + + defp put_redirect_location(request, location_url) do + put_in(request.url, location_url) + end + + defp put_redirect_request_method(request, status) when status in 307..308, do: request + defp put_redirect_request_method(request, _), do: %{request | method: :get} + + defp remove_credentials_if_untrusted(request, true, _), do: request + + defp remove_credentials_if_untrusted(request, _, location_url) do + if {location_url.host, location_url.scheme, location_url.port} == + {request.url.host, request.url.scheme, request.url.port} do + request + else + remove_credentials(request) + end + end + + defp remove_credentials(request) do + headers = List.keydelete(request.headers, "authorization", 0) + request = update_in(request.options, &Map.delete(&1, :auth)) + %{request | headers: headers} + end + + defp remove_params(request) do + update_in(request.options, &Map.delete(&1, :params)) + end + + @doc """ + Downcase response headers names. + """ + def downcase_headers({request, response}) when is_exception(response) do + {request, response} + end + + def downcase_headers({request, response}) do + headers = for {name, value} <- response.headers, do: {prepare_header_name(name), value} + {request, %{response | headers: headers}} + end + + @doc """ + Decompresses the response body based on the `content-encoding` header. + """ + def decompress_body(request_response) + + def decompress_body({request, response}) + when request.options.raw == true or + response.body == "" or + not is_binary(response.body) do + {request, response} + end + + def decompress_body({request, response}) do + compression_algorithms = get_content_encoding_header(response.headers) + {request, update_in(response.body, &decompress_body(&1, compression_algorithms))} + end + + defp decompress_body(body, algorithms) do + Enum.reduce(algorithms, body, &decompress_with_algorithm(&1, &2)) + end + + defp decompress_with_algorithm(gzip, body) when gzip in ["gzip", "x-gzip"] do + :zlib.gunzip(body) + end + + defp decompress_with_algorithm("deflate", body) do + :zlib.unzip(body) + end + + defp decompress_with_algorithm("identity", body) do + body + end + + defp decompress_with_algorithm(algorithm, _body) do + raise("unsupported decompression algorithm: #{inspect(algorithm)}") + end + + @default_retry_delay :timer.seconds(2) + + @doc """ + Retries a request in face of errors. + + This function can be used as either or both response and error step. + + ## Request Options + + * `:retry` - can be one of the following: + + * `:safe` (default) - retry GET/HEAD requests on HTTP 408/429/5xx + responses or exceptions + + * `:always` - always retry + + * `:condition_step` - step on the execution of which depends on whether + to repeat the request + + * `:delay` - sleep this number of milliseconds before making another + attempt, defaults to `#{@default_retry_delay}`. If the response is + HTTP 429 and contains the `retry-after` header, the value of the header + is used as the next retry delay. + + * `:max_retries` - maximum number of retry attempts, defaults to `2` + (for a total of `3` requests to the server, including the initial one.) + + """ + def retry({request, exception}) when is_exception(exception) do + retry(request, exception) + end + + def retry({request, response}) + when not is_map_key(request.options, :retry) or request.options.retry == :safe do + retry_safe(request, response) + end + + def retry({request, response}) when request.options.retry == :always do + retry(request, response) + end + + def retry({request, response}) do + default_condition = fn {_request, response} -> response.status >= 500 end + condition_step = get_options(request.options.retry, :condition_step, default_condition) + + if Request.run_step(condition_step, {request, response}) do + retry(request, response) + else + {request, response} + end + end + + defp retry_safe(request, response_or_exception) do + if request.method in [:get, :head] do + case response_or_exception do + %Response{status: status} when status in [408, 429] or status in 500..599 -> + retry(request, response_or_exception) + + %Response{} -> + {request, response_or_exception} + + exception when is_exception(exception) -> + retry(request, response_or_exception) + end + else + {request, response_or_exception} + end + end + + defp retry(request, response_or_exception) do + retry_count = Request.get_private(request, :retry_count, 0) + + case configure_retry(request, response_or_exception, retry_count) do + %{retry?: true} = retry_params -> + log_retry(response_or_exception, retry_count, retry_params) + Process.sleep(retry_params.delay) + request = Request.put_private(request, :retry_count, retry_count + 1) + + {_, result} = Request.run(request) + {Request.halt(request), result} + + _ -> + {request, response_or_exception} + end + end + + defp configure_retry(request, response_or_exception, retry_count) do + retry_options = get_options(request.options, :retry) + + case get_retry_delay(retry_options, response_or_exception) do + delay when is_integer(delay) -> + max_retries = get_options(retry_options, :max_retries, 2) + retry = retry_count < max_retries + %{delay: delay, max_retries: max_retries, retry?: retry, type: :linear} + + {:retry_after, delay} -> + %{delay: delay, retry?: true, type: :retry_after} + + :exponent -> + max_retries = get_options(retry_options, :max_retries, 30) + max_cap = get_options(retry_options, :max_cap, :timer.minutes(20)) + delays = cap(exponential_backoff(), max_cap) + retry = retry_count < max_retries + %{delay: Enum.at(delays, retry_count), retry?: retry, type: :exponent} + + :x_rate_limit -> + delay = check_x_rate_limit(response_or_exception) + %{delay: delay, retry?: true, type: :x_rate_limit} + end + end + + defp check_x_rate_limit(%Response{headers: headers}) do + case get_headers(headers, ["x-ratelimit-reset", "x-ratelimit-remaining"]) do + %{"x-ratelimit-remaining" => "0", "x-ratelimit-reset" => timestamp} -> + get_x_rate_limit_delay(timestamp) + + %{"x-ratelimit-remaining" => "", "x-ratelimit-reset" => timestamp} -> + get_x_rate_limit_delay(timestamp) + + %{"x-ratelimit-reset" => timestamp} = headers when map_size(headers) == 1 -> + get_x_rate_limit_delay(timestamp) + + _headers -> + @default_retry_delay + end + end + + defp check_x_rate_limit(_response_or_exception), do: @default_retry_delay + + defp get_x_rate_limit_delay(timestamp) do + with {timestamp, ""} <- Integer.parse(timestamp), + {:ok, datetime} <- DateTime.from_unix(timestamp), + seconds when seconds > 0 <- DateTime.diff(datetime, DateTime.utc_now()) do + :timer.seconds(seconds) + else + _ -> @default_retry_delay + end + end + + defp get_retry_delay(options, %Response{status: 429, headers: headers}) do + case get_header(headers, "retry-after") do + nil -> get_options(options, :delay, @default_retry_delay) + header_delay -> {:retry_after, retry_delay_in_ms(header_delay)} + end + end + + defp get_retry_delay(options, _response_or_exception) do + get_options(options, :delay, @default_retry_delay) + end + + defp exponential_backoff(initial_delay \\ :timer.seconds(1), factor \\ 2) do + Stream.unfold(initial_delay, fn last_delay -> + {last_delay, round(last_delay * factor)} + end) + end + + defp cap(delays, max) do + Stream.map(delays, fn + delay when delay <= max -> delay + _ -> max + end) + end + + defp retry_delay_in_ms(delay_value) do + case Integer.parse(delay_value) do + {seconds, ""} -> + :timer.seconds(seconds) + + :error -> + delay_value + |> parse_http_datetime() + |> DateTime.diff(DateTime.utc_now(), :millisecond) + |> max(0) + end + end + + defp log_retry(response_or_exception, retry_count, retry_params) do + message = + case retry_params do + %{type: :retry_after} -> + "Will retry after #{retry_params.delay}ms" + + %{type: :exponent} -> + "Will retry in #{retry_params.delay}ms, retry count: #{retry_count}" + + %{type: :x_rate_limit} -> + "Will retry after #{retry_params.delay}ms" + + %{max_retries: max_retries} when max_retries - retry_count == 1 -> + "Will retry in #{retry_params.delay}ms, 1 attempt left" + + _retry_params -> + attempts = retry_params.max_retries - retry_count + "Will retry in #{retry_params.delay}ms, #{attempts} attempts left" + end + + case response_or_exception do + exception when is_exception(exception) -> + Logger.error(["retry: got exception. ", message]) + Logger.error(["** (#{inspect(exception.__struct__)}) ", Exception.message(exception)]) + + response -> + Logger.error(["retry: got response with status #{response.status}. ", message]) + end + end + + @doc false + def log_response_end({request, response_or_exception}) do + start_time = request.private.request_start_time + metadata = %{adapter: request.adapter, method: request.method, url: to_string(request.url)} + Telemetry.stop(:request, start_time, enrich_metadata(metadata, response_or_exception)) + {request, response_or_exception} + end + + defp enrich_metadata(metadata, exception) when is_exception(exception) do + Map.put(metadata, :error, Exception.message(exception)) + end + + defp enrich_metadata(metadata, response) do + metadata + |> Map.put(:headers, response.headers) + |> Map.put(:status_code, response.status) + end + + ## Utilities + + defp get_options(options, key, default \\ []) + + defp get_options(options, key, default) when is_map_key(options, key) do + Map.get(options, key, default) + end + + defp get_options(options, key, default), do: options[key] || default + + defp get_content_encoding_header(headers) do + headers + |> Enum.flat_map(fn {name, value} -> + if String.downcase(name) == "content-encoding" do + value + |> String.downcase() + |> String.split(",", trim: true) + |> Stream.map(&String.trim/1) + else + [] + end + end) + |> Enum.reverse() + end + + defp get_headers(headers, keys) when is_list(keys) do + headers + |> Keyword.take(keys) + |> Map.new() + end + + defp get_header(headers, name, default_value \\ nil) do + Enum.find_value(headers, default_value, fn {key, value} -> + if String.downcase(key) == name, do: value + end) + end + + defp put_new_header(struct, name, value) do + if Enum.any?(struct.headers, fn {key, _} -> String.downcase(key) == name end) do + struct + else + put_header(struct, name, value) + end + end + + defp put_header(struct, name, value) do + update_in(struct.headers, &[{name, value} | &1]) + end + + defp prepare_header_name(name) when is_atom(name) do + name + |> Atom.to_string() + |> String.replace("_", "-") + |> String.downcase() + end + + defp prepare_header_name(name) when is_binary(name), do: String.downcase(name) + + defp prepare_header_value(%NaiveDateTime{} = naive_datetime), do: naive_datetime + + defp prepare_header_value(%DateTime{} = datetime) do + datetime + |> DateTime.shift_zone!("Etc/UTC") + |> format_http_datetime() + end + + defp prepare_header_value(value), do: String.Chars.to_string(value) + + defp format_http_datetime(datetime) do + Calendar.strftime(datetime, "%a, %d %b %Y %H:%M:%S GMT") + end + + @month_numbers %{ + "Jan" => "01", + "Feb" => "02", + "Mar" => "03", + "Apr" => "04", + "May" => "05", + "Jun" => "06", + "Jul" => "07", + "Aug" => "08", + "Sep" => "09", + "Oct" => "10", + "Nov" => "11", + "Dec" => "12" + } + defp parse_http_datetime(datetime) do + [_day_of_week, day, month, year, time, "GMT"] = String.split(datetime, " ") + date = year <> "-" <> @month_numbers[month] <> "-" <> day + + case DateTime.from_iso8601(date <> " " <> time <> "Z") do + {:ok, valid_datetime, 0} -> + valid_datetime + + {:error, reason} -> + raise "could not parse \"Retry-After\" header #{datetime} - #{reason}" + end + end +end diff --git a/lib/http_client/telemetry.ex b/lib/http_client/telemetry.ex index 7f1c2cc..8016e9c 100644 --- a/lib/http_client/telemetry.ex +++ b/lib/http_client/telemetry.ex @@ -13,8 +13,9 @@ defmodule HTTPClient.Telemetry do #### Metadata: * `:adapter` - The name of adapter impelementation. - * `:args` - The arguments passed in the request (url, headers, etc.). + * `:headers` - The headers passed in the request. * `:method` - The method used in the request. + * `:url` - The requested url. * `[:http_client, :request, :stop]` - Executed after a request is finished. @@ -23,8 +24,9 @@ defmodule HTTPClient.Telemetry do #### Metadata: * `:adapter` - The name of adapter impelementation. - * `:args` - The arguments passed in the request (url, headers, etc.). + * `:headers` - The headers passed in the response. * `:method` - The method used in the request. + * `:url` - The requested url. * `:status_code` - This value is optional. The response status code. * `:error` - This value is optional. It includes any errors that occured while making the request. """ diff --git a/mix.exs b/mix.exs index ae52f75..2e78f33 100644 --- a/mix.exs +++ b/mix.exs @@ -2,7 +2,7 @@ defmodule HTTPClient.MixProject do use Mix.Project @name "HTTPClient" - @version "0.3.10" + @version "0.4.0" @repo_url "https://github.com/ChannexIO/http_client" def project do @@ -33,10 +33,11 @@ defmodule HTTPClient.MixProject do {:finch, "~> 0.20"}, {:telemetry, "~> 1.3"}, {:jason, "~> 1.4"}, + {:mime, "~> 2.0"}, {:plug, "~> 1.19", only: :test, override: true}, {:plug_cowboy, "~> 2.7", only: :test, override: true}, {:lasso, "~> 0.1", only: :test}, - {:ex_doc, "~> 0.39", only: :dev, runtime: false} + {:ex_doc, ">= 0.0.0", only: :dev, runtime: false} ] end diff --git a/mix.lock b/mix.lock index f3b7b3c..3e3b9a1 100644 --- a/mix.lock +++ b/mix.lock @@ -1,11 +1,9 @@ %{ - "bandit": {:hex, :bandit, "1.10.0", "f8293b4a4e6c06b31655ae10bd3462f59d8c5dbd1df59028a4984f10c5961147", [:mix], [{:hpax, "~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "43ebceb7060a4d8273e47d83e703d01b112198624ba0826980caa3f5091243c4"}, - "castore": {:hex, :castore, "0.1.16", "2675f717adc700475345c5512c381ef9273eb5df26bdd3f8c13e2636cf4cc175", [:mix], [], "hexpm", "28ed2c43d83b5c25d35c51bc0abf229ac51359c170cba76171a462ced2e4b651"}, + "bandit": {:hex, :bandit, "1.10.1", "6b1f8609d947ae2a74da5bba8aee938c94348634e54e5625eef622ca0bbbb062", [:mix], [{:hpax, "~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "4b4c35f273030e44268ace53bf3d5991dfc385c77374244e2f960876547671aa"}, "certifi": {:hex, :certifi, "2.15.0", "0e6e882fcdaaa0a5a9f2b3db55b1394dba07e8d6d9bcad08318fb604c6839712", [:rebar3], [], "hexpm", "b147ed22ce71d72eafdad94f055165c1c182f61a2ff49df28bcc71d1d5b94a60"}, "cowboy": {:hex, :cowboy, "2.14.2", "4008be1df6ade45e4f2a4e9e2d22b36d0b5aba4e20b0a0d7049e28d124e34847", [:make, :rebar3], [{:cowlib, ">= 2.16.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, ">= 1.8.0 and < 3.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "569081da046e7b41b5df36aa359be71a0c8874e5b9cff6f747073fc57baf1ab9"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, "cowlib": {:hex, :cowlib, "2.16.0", "54592074ebbbb92ee4746c8a8846e5605052f29309d3a873468d76cdf932076f", [:make, :rebar3], [], "hexpm", "7f478d80d66b747344f0ea7708c187645cfcc08b11aa424632f78e25bf05db51"}, - "earmark": {:hex, :earmark, "1.4.4", "4821b8d05cda507189d51f2caeef370cf1e18ca5d7dfb7d31e9cafe6688106a4", [:mix], [], "hexpm", "1f93aba7340574847c0f609da787f0d79efcab51b044bb6e242cae5aca9d264d"}, "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, "ex_doc": {:hex, :ex_doc, "0.39.3", "519c6bc7e84a2918b737aec7ef48b96aa4698342927d080437f61395d361dcee", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "0590955cf7ad3b625780ee1c1ea627c28a78948c6c0a9b0322bd976a079996e1"}, "finch": {:hex, :finch, "0.20.0", "5330aefb6b010f424dcbbc4615d914e9e3deae40095e73ab0c1bb0968933cadf", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2658131a74d051aabfcba936093c903b8e89da9a1b63e430bee62045fa9b2ee2"}, @@ -17,7 +15,7 @@ "lasso": {:hex, :lasso, "0.1.4", "c4ae258fb7d0ef5ae217822b4d90c34cda2d9389ebb793330184d8bc626f8617", [:mix], [{:bandit, "~> 1.0", [hex: :bandit, repo: "hexpm", optional: false]}], "hexpm", "24a3bbf06e03508b4c81cb257a47809b81e6e81ed12773c23ef0f86d0b87d393"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, - "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"}, + "makeup_erlang": {:hex, :makeup_erlang, "1.0.3", "4252d5d4098da7415c390e847c814bad3764c94a814a0b4245176215615e1035", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "953297c02582a33411ac6208f2c6e55f0e870df7f80da724ed613f10e6706afd"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, "mimerl": {:hex, :mimerl, "1.4.0", "3882a5ca67fbbe7117ba8947f27643557adec38fa2307490c4c4207624cb213b", [:rebar3], [], "hexpm", "13af15f9f68c65884ecca3a3891d50a7b57d82152792f3e19d88650aa126b144"}, @@ -29,7 +27,7 @@ "plug": {:hex, :plug, "1.19.1", "09bac17ae7a001a68ae393658aa23c7e38782be5c5c00c80be82901262c394c0", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "560a0017a8f6d5d30146916862aaf9300b7280063651dd7e532b8be168511e62"}, "plug_cowboy": {:hex, :plug_cowboy, "2.7.5", "261f21b67aea8162239b2d6d3b4c31efde4daa22a20d80b19c2c0f21b34b270e", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "20884bf58a90ff5a5663420f5d2c368e9e15ed1ad5e911daf0916ea3c57f77ac"}, "plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"}, - "ranch": {:hex, :ranch, "1.8.1", "208169e65292ac5d333d6cdbad49388c1ae198136e4697ae2f474697140f201c", [:make, :rebar3], [], "hexpm", "aed58910f4e21deea992a67bf51632b6d60114895eb03bb392bb733064594dd0"}, + "ranch": {:hex, :ranch, "2.2.0", "25528f82bc8d7c6152c57666ca99ec716510fe0925cb188172f41ce93117b1b0", [:make, :rebar3], [], "hexpm", "fa0b99a1780c80218a4197a59ea8d3bdae32fbff7e88527d7d8a4787eff4f8e7"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, "thousand_island": {:hex, :thousand_island, "1.4.3", "2158209580f633be38d43ec4e3ce0a01079592b9657afff9080d5d8ca149a3af", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6e4ce09b0fd761a58594d02814d40f77daff460c48a7354a15ab353bb998ea0b"}, diff --git a/test/http_client_test.exs b/test/http_client_test.exs index e2bbcfb..59de2d9 100644 --- a/test/http_client_test.exs +++ b/test/http_client_test.exs @@ -3,7 +3,7 @@ defmodule HTTPClientTest do doctest HTTPClient - alias HTTPClient.Response + alias HTTPClient.{Error, Response} setup do {:ok, lasso: Lasso.open()} @@ -33,6 +33,14 @@ defmodule HTTPClientTest do TestFinchRequest.get(endpoint(lasso), headers, options) end + @tag :skip + test "get/3 error response", %{lasso: lasso} do + # Lasso.down(lasso) + + assert {:error, %Error{reason: "connection refused"}} == + TestFinchRequest.get(endpoint(lasso), [], []) + end + test "post/4 success response", %{lasso: lasso} do req_body = ~s({"response":"please"}) response_body = ~s({"right":"here"}) @@ -52,12 +60,20 @@ defmodule HTTPClientTest do end) headers = [{"content-type", "application/json"}] - options = [params: %{a: 1, b: 2}, basic_auth: {"username", "password"}] + options = [params: %{a: 1, b: 2}, auth: {:basic, {"username", "password"}}] assert {:ok, %Response{status: 200, body: ^response_body}} = TestFinchRequest.post(endpoint(lasso), req_body, headers, options) end + @tag :skip + test "post/4 error response", %{lasso: lasso} do + # Lasso.down(lasso) + + assert {:error, %Error{reason: "connection refused"}} == + TestFinchRequest.post(endpoint(lasso), "{}", [], []) + end + test "request/5 success response", %{lasso: lasso} do Lasso.expect_once(lasso, "DELETE", "/", fn conn -> Plug.Conn.send_resp(conn, 200, "OK") @@ -66,6 +82,14 @@ defmodule HTTPClientTest do assert {:ok, %Response{status: 200, body: "OK"}} = TestFinchRequest.request(:delete, endpoint(lasso), "", [], []) end + + @tag :skip + test "request/5 error response", %{lasso: lasso} do + # Lasso.down(lasso) + + assert {:error, %Error{reason: "connection refused"}} == + TestFinchRequest.request(:post, endpoint(lasso), "{}", [], []) + end end describe "telemetry" do @@ -89,27 +113,23 @@ defmodule HTTPClientTest do assert is_integer(measurements.system_time) assert meta.adapter == HTTPClient.Adapters.HTTPoison - assert meta.args == [ - endpoint(lasso), - [{"content-type", "application/json"}], - [params: %{a: 1, b: 2}, basic_auth: {"username", "password"}] + assert meta.headers == [ + {"authorization", "Basic dXNlcm5hbWU6cGFzc3dvcmQ="}, + {"accept-encoding", "gzip"}, + {"content-type", "application/json"} ] assert meta.method == :get + assert URI.new!(meta.url) == URI.new!(endpoint(lasso, "/?a=1&b=2")) send(parent, {ref, :start}) [:http_client, :request, :stop] -> assert is_integer(measurements.duration) assert meta.adapter == HTTPClient.Adapters.HTTPoison - - assert meta.args == [ - endpoint(lasso), - [{"content-type", "application/json"}], - [params: %{a: 1, b: 2}, basic_auth: {"username", "password"}] - ] - + assert is_list(meta.headers) assert meta.method == :get assert meta.status_code == 200 + assert URI.new!(meta.url) == URI.new!(endpoint(lasso, "/?a=1&b=2")) send(parent, {ref, :stop}) _ -> @@ -128,7 +148,7 @@ defmodule HTTPClientTest do ) headers = [{"content-type", "application/json"}] - options = [params: %{a: 1, b: 2}, basic_auth: {"username", "password"}] + options = [params: %{a: 1, b: 2}, auth: {:basic, {"username", "password"}}] assert {:ok, %{status: 200}} = TestDefaultRequest.get(endpoint(lasso), headers, options) assert_receive {^ref, :start}