diff --git a/developer-portal/content/reference/index.md b/developer-portal/content/reference/index.md index e15f25597b..2c6f31cd59 100644 --- a/developer-portal/content/reference/index.md +++ b/developer-portal/content/reference/index.md @@ -252,6 +252,7 @@ in the `5xx` range indicate an error with Fox's platform. + @@ -260,3 +261,4 @@ in the `5xx` range indicate an error with Fox's platform. + diff --git a/developer-portal/content/reference/objects/cms.apib b/developer-portal/content/reference/objects/cms.apib new file mode 100644 index 0000000000..bd8c4f1456 --- /dev/null +++ b/developer-portal/content/reference/objects/cms.apib @@ -0,0 +1,41 @@ +# Data Structures + +## ContentTypeBase ++ schema: `{ "title": {"type": ["string"], "required": true }}` (required, string) - JSON schema of a content_type ++ name: `BlogPost` (required, string) - Name of a content_type + +## ContentTypePayload +- Include ContentTypeBase + +## ContentTypeUpdatePayload ++ schema: `{ "another_title": {"type": ["string"], "required": true }}` (optional, string) - JSON schema of a content_type ++ name: `BlogPost` (optional, string) - Name of a content_type + +## ContentTypeResponse +- Include ContentTypeBase ++ scope: `1` (required, string) - Scope of an admin who created a given ContentType ++ id: `1` (required, number) - ID of a created ContentType ++ created_by: `4` (required, number) - ID of a user who created a given ContentType ++ versions: `["2017-06-12T03:13:46Z", "2017-06-13T03:12:46Z"]` (optional, string) - Previous versions of ContentType + +## EntityBase ++ content_type_id: 1 (required, number) - id of a corresponding content_type ++ storefront: `theperfectgourmet.com` (required, string) - Name of a store on which given Entity has been created ++ content: ` { "title": "foo" } ` (required, string) - JSON content of a Entity + +## EntityPayload +- Include EntityBase + +## EntityUpdatePayload ++ content_type_id: 1 (optional, number) - id of a corresponding content_type ++ content: ` { "title": "bar" } ` (optional, string) - JSON content of a Entity + +## EntityAdminResponse +- Include EntityBase ++ schema_version: `2017-06-12T03:13:46Z` (required, string) - Version of schema Entity was validated against ++ storefront: `theperfectgourmet.com` (required, string) - Name of a store on which given Entity has been created ++ kind: `BlogPost` (required, string) - Type of a created Entity ++ id: `3` (required, number) - ID of a created Entity ++ created_by: `4` (required, number) - ID of a user who created a given Entity ++ versions: `["2017-06-12T03:13:46Z", "2017-06-13T03:12:46Z"]` (optional, string) - Previous versions of Entity + diff --git a/developer-portal/content/reference/resources/admin_cms.apib b/developer-portal/content/reference/resources/admin_cms.apib new file mode 100644 index 0000000000..70058babd5 --- /dev/null +++ b/developer-portal/content/reference/resources/admin_cms.apib @@ -0,0 +1,105 @@ +## ContentType [/v1/admin/content_types{id}] + +This object represents ContentTypes in CMS. +Each ContentType describes specific Entity. + ++ Parameters + + id: `1` (required, number) - ContentType ID. + ++ Model (application/json) + + Attributes (ContentTypeResponse) + + +### Retrieve ContentType [GET] + +Allows an administrator to retrieve a content_type record, which includes all +information about the content_type. + ++ Response 200 (application/json) + + Attributes (ContentTypeResponse) + +### Create New ContentType [POST /v1/admin/content_types] + +Allows an administrator to create a new content_types. + ++ Request (application/json) + + Attributes (ContentTypePayload) ++ Response 200 (application/json) + + Attributes (ContentTypeResponse) + +### Update Existing ContentType [PUT] + ++ Request (application/json) + + Attributes(ContentTypeUpdatePayload) ++ Response 200 (application/json) + + Attributes (ContentTypeResponse) + +### Get specific version of ContentType [GET /v1/admin/content_types/{id}/versions?ver={version}] + ++ Parameters + + id: `1` (required, number) - ContentType ID. + + ver: `2017-06-13T16:14:51.974673Z` (required, string) - Version of a ContentType + ++ Response 200 (application/json) + + Attributes (ContentTypeResponse) + +### Restore specific version of ContentType [PUT /v1/admin/content_types/{id}/restore?ver={version}] + ++ Parameters + + id: `1` (required, number) - ContentType ID. + + ver: `2017-06-13T16:14:51.974673Z` (required, string) - Version of a ContentType + ++ Response 200 (application/json) + + Attributes (ContentTypeResponse) + +## Entity [/v1/admin/entities{id}] + +This object describes an actual entity of CMS: Blog posts, testimonials, etc. +Each Entity validates against corresponding ContentType's schema before creation. + ++ Parameters + + id: `1` (required, number) - Entity ID. + ++ Model (application/json) + + Attributes (EntityAdminResponse) + +### Retrieve Entity [GET] + +Allows an administrator to retrieve a single Entity record. + ++ Response 200 (application/json) + + Attributes (EntityAdminResponse) + +### Create New Entity [POST /v1/admin/entities] + +Allows an administrator to create a new Entity. + ++ Request (application/json) + + Attributes (EntityPayload) ++ Response 200 (application/json) + + Attributes (EntityAdminResponse) + +### Update Existing Entity [PUT] + ++ Request (application/json) + + Attributes(EntityUpdatePayload) ++ Response 200 (application/json) + + Attributes (EntityAdminResponse) + +### Get specific version of Entity [GET /v1/admin/entities/{id}/versions?ver={version}] + ++ Parameters + + id: `1` (required, number) - Entity ID. + + ver: `2017-06-13T16:14:51.974673Z` (required, string) - Version of a Entity + ++ Response 200 (application/json) + + Attributes (EntityAdminResponse) + +### Restore specific version of Entity [PUT /v1/admin/entities/{id}/restore?ver={version}] + ++ Parameters + + id: `1` (required, number) - Entity ID. + + ver: `2017-06-13T16:14:51.974673Z` (required, string) - Version of a Entity + ++ Response 200 (application/json) + + Attributes (EntityAdminResponse) \ No newline at end of file diff --git a/geronimo/Makefile b/geronimo/Makefile index 7df18405af..57acb64e4f 100644 --- a/geronimo/Makefile +++ b/geronimo/Makefile @@ -33,8 +33,7 @@ test: MIX_ENV=test mix espec seed: - mix run priv/seeds/clothes_accessories_categories.exs - mix run priv/seeds/clothes_schema.exs + mix run priv/seeds/seeds.exs #################################################################### # Docker build targets # @@ -82,6 +81,7 @@ migrate: reset: @make drop-db + dropdb --if-exists $(DB_TEST) -U $(DB_USER) @make drop-user @make create-user @make create-db diff --git a/geronimo/README.md b/geronimo/README.md index d249d0ea03..9ee1709f1f 100644 --- a/geronimo/README.md +++ b/geronimo/README.md @@ -51,6 +51,18 @@ GERONIMO_DB_HOST=localhost GERONIMO_DB_USER=geronimo GERONIMO_DB_NAME=geronimo_development GERONIMO_DB_PASSWORD='' + +# kafka +BROKER_HOST=kafka_broker_host +BROKER_PORT=9092 +CONSUMER_GROUP=geronimo_kafka_ex +SCHEMA_REGISTRY_IP=schema_registry_ip +SCHEMA_REGISTRY_PORT=8081 + +# Start kafka worker on application start +START_WORKER=true + +#jwt PUBLIC_KEY=/path/to/public_key.pem ``` @@ -488,6 +500,8 @@ Params: |----|----|-----------|---------| |content_type_id|Integer|Corresponding ContentType id|Yes| |content|JSON|Content of an Entity|Yes| +|storefront|string|Storefront on which given entity has been created|Yes| + **Request:** `POST /v1/admin/entities/` @@ -495,6 +509,7 @@ Params: ```json { "content_type_id": 1, + "storefront": "foo.bar", "content": { "title":"Some title foooooo", "body":"Lorem ipsum", @@ -511,6 +526,7 @@ Params: "updated_at": "2017-06-12T03:15:17Z", "scope": "1", "schema_version": "2017-06-12T03:13:46Z", + "storefront": "foo.bar", "kind": "BlogPost", "inserted_at": "2017-06-12T03:15:17Z", "id": 3, diff --git a/geronimo/config/test.exs b/geronimo/config/test.exs index 8a3bf43836..5fde98feef 100644 --- a/geronimo/config/test.exs +++ b/geronimo/config/test.exs @@ -6,4 +6,13 @@ config :geronimo, Geronimo.Repo, database: System.get_env("GERONIMO_DB_NAME"), hostname: System.get_env("GERONIMO_DB_HOST"), pool: Ecto.Adapters.SQL.Sandbox, - types: Geronimo.PostgresTypes \ No newline at end of file + types: Geronimo.PostgresTypes + +config :exvcr, [ + vcr_cassette_library_dir: "spec/fixture/vcr_cassettes", + filter_sensitive_data: [ + [pattern: ".+", placeholder: "PASSWORD_PLACEHOLDER"] + ], + filter_url_params: false, + response_headers_blacklist: [] +] \ No newline at end of file diff --git a/geronimo/lib/geronimo/api/router/v1/admin.ex b/geronimo/lib/geronimo/api/router/v1/admin.ex index 41e77c0b7e..4d85baa443 100644 --- a/geronimo/lib/geronimo/api/router/v1/admin.ex +++ b/geronimo/lib/geronimo/api/router/v1/admin.ex @@ -105,6 +105,7 @@ defmodule Geronimo.Router.V1.Admin do params do requires :content_type_id, type: Integer requires :content, type: Any + requires :storefront, type: String end post do @@ -112,7 +113,7 @@ defmodule Geronimo.Router.V1.Admin do {:ok, content_type} = ContentType.get(params[:content_type_id], conn.assigns[:current_user].scope) validated = Validation.validate!(params[:content], content_type.schema) - case Entity.create(validated, content_type, conn.assigns[:current_user]) do + case Entity.create(validated, content_type, params[:storefront], conn.assigns[:current_user]) do {:ok, record} -> respond_with(conn, record) {:error, err} -> respond_with(conn, err, 400) end diff --git a/geronimo/lib/geronimo/kafka/pusher.ex b/geronimo/lib/geronimo/kafka/pusher.ex new file mode 100644 index 0000000000..ba6847caa4 --- /dev/null +++ b/geronimo/lib/geronimo/kafka/pusher.ex @@ -0,0 +1,27 @@ +defmodule Geronimo.Kafka.Pusher do + @moduledoc """ + Implements sync and async pushes to Kafka + """ + require Logger + + def push(module, obj) do + kind = apply(module, :table, []) + data = apply(module, :avro_encode!, [obj]) + res = KafkaEx.produce("geronimo_#{kind}", 0, data, + key: "#{kind}_#{obj.id}", + worker_name: :geronimo_worker) + Logger.debug "#{Inflex.singularize(kind)} with id #{obj.id} pushed to kafka #{inspect(res)}" + end + + def push_async(kind, obj) do + unless Mix.env == :test do + Task.async(fn -> + push(kind, obj) + end) + end + end + + def push_async_await(kind, obj) do + push_async(kind, obj) |> Task.await + end +end diff --git a/geronimo/lib/geronimo/kafka/schema_registry_client.ex b/geronimo/lib/geronimo/kafka/schema_registry_client.ex index 91050e3f9a..ceb4fa6d38 100644 --- a/geronimo/lib/geronimo/kafka/schema_registry_client.ex +++ b/geronimo/lib/geronimo/kafka/schema_registry_client.ex @@ -1,36 +1,38 @@ defmodule Geronimo.Kafka.SchemaRegistryClient do + @moduledoc """ + Provides a easy and conveniens functions to get/store schemas in schema registry + """ use HTTPoison.Base def get_schema(object, id) do - case get("/subjects/#{object}/versions/#{id}") do - {:ok, %HTTPoison.Response{body: body, headers: _, status_code: 200}} -> - body[:schema] - |> Poison.decode! - |> Utils.atomize - {:ok, %HTTPoison.Response{body: body, headers: _, status_code: _}} -> body - {:error, err} -> err - end - end - - def store_schema(schema) do - schema + get("/subjects/#{object}/versions/#{id}") + |> response_body() end - def process_url(url) do - "http://#{schema_registry_url()}/#{url}" + def store_schema(object, schema) do + post("/subjects/#{object}/versions", Poison.encode!(%{schema: schema})) + |> response_body() end - def process_response_body(body) do - body - |> Poison.decode! - |> Enum.map(fn({k, v}) -> {String.to_atom(k), v} end) + defp response_body(response) do + case response do + {:ok, %HTTPoison.Response{body: body, headers: _, status_code: 200}} -> {:ok, body} + {:ok, %HTTPoison.Response{body: body, headers: _, status_code: _}} -> {:error, body} + {:error, err} -> {:fail, err.reason} + end end - def schema_registry_url do - url = Application.fetch_env!(:geronimo, :schema_registry_ip) + def process_url(path) do + ip = Application.fetch_env!(:geronimo, :schema_registry_ip) port = Application.fetch_env!(:geronimo, :schema_registry_port) - "http://#{url}:#{port}" + "http://#{ip}:#{port}" <> path end def process_request_headers(), do: ["Content-Type", "application/vnd.schemaregistry.v1+json"] + + def process_response_body(body) do + body + |> Poison.decode! + |> Utils.atomize + end end diff --git a/geronimo/lib/geronimo/kafka/worker.ex b/geronimo/lib/geronimo/kafka/worker.ex index 8795b10fdd..76c6b7e802 100644 --- a/geronimo/lib/geronimo/kafka/worker.ex +++ b/geronimo/lib/geronimo/kafka/worker.ex @@ -1,27 +1,32 @@ defmodule Geronimo.Kafka.Worker do + @moduledoc """ + Starts KafkaEs worker on app start and registers all needed schemas + NB: Add new modules to register_schemas() if needed. + """ + require Logger + alias Geronimo.Kafka.SchemaRegistryClient def start do kafka_url = [{Application.fetch_env!(:geronimo, :kafka_host), Application.fetch_env!(:geronimo, :kafka_port) |> String.to_integer }] KafkaEx.create_worker(:geronimo_worker, [uris: kafka_url, consumer_group: Application.fetch_env!(:geronimo, :consumer_group)]) + register_schemas() end - def push(kind, obj) do - KafkaEx.produce("geronimo_#{kind}", 0, Poison.encode!(obj), - key: "#{kind}_#{obj.id}", worker_name: :geronimo_worker) - end + def register_schemas do + Task.async(fn-> + modules = [Geronimo.ContentType, Geronimo.Entity] - def push_async(kind, obj) do - unless Mix.env == :test do - Task.async(fn -> - push(kind, obj) + Enum.each(modules, fn(module) -> + key_schema = apply(module, :avro_schema_key, []) + value_schema = apply(module, :avro_schema_value, []) + object = apply(module, :table, []) + {:ok, k_res} = SchemaRegistryClient.store_schema("#{object}-key", key_schema) + {:ok, v_ver} = SchemaRegistryClient.store_schema("#{object}-value", value_schema) + Logger.info "Schemas for #{object} registered. Key: #{inspect(k_res)}, value: #{inspect(v_ver)}" end) - end - end - - def push_async_await(kind, obj) do - push_async(kind, obj) |> Task.await + end) |> Task.await end end diff --git a/geronimo/lib/geronimo/models/entity.ex b/geronimo/lib/geronimo/models/entity.ex index 21220c0ddb..91bd827f7b 100644 --- a/geronimo/lib/geronimo/models/entity.ex +++ b/geronimo/lib/geronimo/models/entity.ex @@ -10,14 +10,16 @@ defmodule Geronimo.Entity do use Geronimo.Kafka.Avro @derive {Poison.Encoder, only: [:id, :kind, :content, :schema_version, - :content_type_id, :created_by, :inserted_at, :updated_at, :versions, :scope]} + :content_type_id, :created_by, :inserted_at, :updated_at, :versions, + :scope, :storefront]} - @required_params [:content, :kind, :content_type_id, :created_by, :schema_version, :scope] + @required_params [:content, :kind, :content_type_id, :created_by, :schema_version, :scope, :storefront] @optional_params [] schema "entities" do field :kind, :string field :content, :map + field :storefront, :string field :schema_version, :utc_datetime field :content_type_id, :integer field :created_by, :integer @@ -32,15 +34,15 @@ defmodule Geronimo.Entity do |> validate_required(@required_params) end - def create({:ok, params}, content_type = %ContentType{}, user = %Geronimo.User{}) do + def create({:ok, params}, content_type = %ContentType{}, storefront, user = %Geronimo.User{}) do prms = Map.merge(%{content: params}, %{schema_version: content_type.updated_at, kind: content_type.name, content_type_id: content_type.id, - created_by: user.id, scope: user.scope}) + created_by: user.id, scope: user.scope, storefront: storefront}) Repo.transaction(fn -> case Repo.insert(changeset(%Geronimo.Entity{}, prms)) do {:ok, record} -> - Geronimo.Kafka.Worker.push_async(table(), record) + Geronimo.Kafka.Pusher.push_async(__MODULE__, record) record {_, changes} -> Repo.rollback(changes) @@ -49,7 +51,7 @@ defmodule Geronimo.Entity do end) end - def create(errors, _, _) when is_list(errors), do: wrap_errors(errors) + def create(errors, _, _, _) when is_list(errors), do: wrap_errors(errors) def update(id, prms = {:ok, params}, user = %Geronimo.User{}) when is_tuple(prms) do {:ok, row} = get(id, user.scope) @@ -58,6 +60,7 @@ defmodule Geronimo.Entity do Repo.transaction(fn -> case Repo.update(changes) do {:ok, record} -> + Geronimo.Kafka.Pusher.push_async(__MODULE__, record) Map.merge(record, %{versions: get_versions(record.id)}) {:error, changeset} -> Repo.rollback(changeset) @@ -69,7 +72,7 @@ defmodule Geronimo.Entity do def update(_id, errors, _user) when is_list(errors), do: wrap_errors(errors) def version_fields do - "id, content, kind, schema_version, content_type_id, inserted_at, updated_at" + "id, content, kind, schema_version, storefront, content_type_id, inserted_at, updated_at" end def content_field, do: :content @@ -88,6 +91,7 @@ defmodule Geronimo.Entity do "kind" => entity.kind, "created_by" => entity.created_by, "id" => entity.id, + "storefront" => entity.storefront, "inserted_at" => Timex.format!(entity.inserted_at, "%FT%T.%fZ", :strftime), "updated_at" => Timex.format!(entity.updated_at, "%FT%T.%fZ", :strftime), "schema_version" => Timex.format!(entity.schema_version, "%FT%T.%fZ", :strftime), diff --git a/geronimo/lib/geronimo/utils/avro.ex b/geronimo/lib/geronimo/utils/avro.ex index e71de05e2e..3cb9521a2b 100644 --- a/geronimo/lib/geronimo/utils/avro.ex +++ b/geronimo/lib/geronimo/utils/avro.ex @@ -9,8 +9,8 @@ defmodule Geronimo.Kafka.Avro do """ def avro_encode(data_id) do data = apply(__MODULE__, :to_avro, [data_id]) - schema = avro_schema() - type = 'com.foxcommerce.geronimo.' ++ String.to_char_list(model_name()) + schema = avro_schema_value() + type = 'com.foxcommerce.geronimo.' ++ String.to_char_list(table()) Avrolixr.Codec.encode(data, schema, type) end @@ -37,20 +37,24 @@ defmodule Geronimo.Kafka.Avro do @doc """ Returns AVRO schema for given model according to Ecto.Schema """ - def avro_schema(as_json \\ true) do + def avro_schema_value(as_json \\ true) do fields = apply(__MODULE__, :__schema__, [:types]) |> Enum.into([]) |> Enum.map(fn {k, v} -> avro_field_type({k, v}) end) - schema = %{fields: fields, name: model_name(), namespace: "com.foxcommerce.geronimo", type: "record"} + schema = %{fields: fields, name: table(), namespace: "com.foxcommerce.geronimo", type: "record"} case as_json do true -> Poison.encode!(schema) _ -> schema end end - defp model_name do - [_a, name] = Inflex.underscore(__MODULE__) |> String.split(".") - name + def avro_schema_key(as_json \\ true) do + fields = [%{"name" => "id","type" =>["null","int"]}] + schema = %{fields: fields, name: "#{table()}_pkey", namespace: "com.foxcommerce.geronimo", type: "record"} + case as_json do + true -> Poison.encode!(schema) + _ -> schema + end end defp avro_field_type({k, v}) do diff --git a/geronimo/lib/geronimo/utils/crud.ex b/geronimo/lib/geronimo/utils/crud.ex index ffe101575e..30a992f7a0 100644 --- a/geronimo/lib/geronimo/utils/crud.ex +++ b/geronimo/lib/geronimo/utils/crud.ex @@ -33,7 +33,7 @@ defmodule Geronimo.Crud do Repo.transaction(fn -> case Repo.insert(changeset(apply(__MODULE__, :__struct__, []), payload)) do {:ok, record} -> - Geronimo.Kafka.Worker.push_async(table(), record) + Geronimo.Kafka.Pusher.push_async(__MODULE__, record) record {_, changes} -> Repo.rollback(changes) end @@ -88,7 +88,9 @@ defmodule Geronimo.Crud do defp apply_change(changeset) do Repo.transaction(fn -> case Repo.update(changeset) do - {:ok, record} -> Map.merge(record, %{versions: get_versions(record.id)}) + {:ok, record} -> + Geronimo.Kafka.Pusher.push_async(__MODULE__, record) + Map.merge(record, %{versions: get_versions(record.id)}) {:error, changeset} -> Repo.rollback(changeset) {:error, changeset} diff --git a/geronimo/lib/geronimo/validation.ex b/geronimo/lib/geronimo/validation.ex index 0dd2431dc9..3fa4964e3e 100644 --- a/geronimo/lib/geronimo/validation.ex +++ b/geronimo/lib/geronimo/validation.ex @@ -33,7 +33,7 @@ defmodule Geronimo.Validation do def valid_date?(d) do case Timex.parse(d, "%FT%T.%fZ", :strftime) do - {:error, _} -> {:error, "should be like 1970-01-01T00:00:00.000000Z"} + {:error, _} -> {:error, "should be like 1970-01-01T00:00:00.0Z"} {:ok, _} -> :ok end end diff --git a/geronimo/mix.exs b/geronimo/mix.exs index 262a08746d..e3e3a6273f 100644 --- a/geronimo/mix.exs +++ b/geronimo/mix.exs @@ -7,7 +7,7 @@ defmodule Geronimo.Mixfile do elixir: "~> 1.4", build_embedded: Mix.env == :prod, start_permanent: Mix.env == :prod, - preferred_cli_env: [espec: :test], + preferred_cli_env: [espec: :test, vcr: :test, "vcr.delete": :test, "vcr.check": :test, "vcr.show": :test], elixirc_paths: elixirc_paths(Mix.env), deps: deps()] end @@ -44,6 +44,7 @@ defmodule Geronimo.Mixfile do {:inflex, "~> 1.8.1" }, {:exsync, "~> 0.1", only: :dev}, {:espec, "~> 1.3.2", only: :test}, - {:ex_machina, "~> 2.0", only: :test}] + {:ex_machina, "~> 2.0", only: :test}, + {:exvcr, "~> 0.8", only: :test}] end end diff --git a/geronimo/mix.lock b/geronimo/mix.lock index 9be974597b..cb1b827474 100644 --- a/geronimo/mix.lock +++ b/geronimo/mix.lock @@ -1,4 +1,5 @@ %{"avrolixr": {:git, "https://github.com/retgoat/avrolixr", "410944ad213fc856c0c55b7603295ec96e605c56", []}, + "bypass": {:hex, :bypass, "0.6.0", "fd0a8004fada4464e2ba98497755310b892a097f2fd975f4f787cf264066a335", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "certifi": {:hex, :certifi, "1.2.1", "c3904f192bd5284e5b13f20db3ceac9626e14eeacfbb492e19583cf0e37b22be", [:rebar3], [], "hexpm"}, "combine": {:hex, :combine, "0.9.6", "8d1034a127d4cbf6924c8a5010d3534d958085575fa4d9b878f200d79ac78335", [:mix], [], "hexpm"}, "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"}, @@ -12,14 +13,19 @@ "erlavro": {:git, "https://github.com/avvo/erlavro", "fb7c7f0b2784468edeea0961984247242097aee5", []}, "espec": {:hex, :espec, "1.3.4", "8a67f6530dcc92a235af7767f6a323aa81c854ee069b28bc2340e54ab88468f9", [:mix], [{:meck, "0.8.4", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, "ex_machina": {:hex, :ex_machina, "2.0.0", "ec284c6f57233729cea9319e083f66e613e82549f78eccdb2059aeba5d0df9f3", [:mix], [{:ecto, "~> 2.1", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, + "exactor": {:hex, :exactor, "2.2.3", "a6972f43bb6160afeb73e1d8ab45ba604cd0ac8b5244c557093f6e92ce582786", [:mix], [], "hexpm"}, "exfswatch": {:hex, :exfswatch, "0.4.2", "d88a63b5c2f8f040230d22010588ff73286fd1aef32564115afa3051eaa4391d", [:mix], [], "hexpm"}, + "exjsx": {:hex, :exjsx, "3.2.1", "1bc5bf1e4fd249104178f0885030bcd75a4526f4d2a1e976f4b428d347614f0f", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm"}, "exsync": {:hex, :exsync, "0.1.4", "f5800f5c3137271bf7c0f5ca623919434f91798e1be1b9d50fc2c59168d44f17", [:mix], [{:exfswatch, "~> 0.4", [hex: :exfswatch, repo: "hexpm", optional: false]}], "hexpm"}, + "exvcr": {:hex, :exvcr, "0.8.10", "17090dea4758eb2349146746084a7c422c77f36b922611df6a46fef40a4bf94c", [:mix], [{:exactor, "~> 2.2", [hex: :exactor, repo: "hexpm", optional: false]}, {:exjsx, "~> 3.2", [hex: :exjsx, repo: "hexpm", optional: false]}, {:httpoison, "~> 0.11", [hex: :httpoison, repo: "hexpm", optional: true]}, {:httpotion, "~> 3.0", [hex: :httpotion, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.2.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:meck, "~> 0.8.3", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, "gettext": {:hex, :gettext, "0.13.1", "5e0daf4e7636d771c4c71ad5f3f53ba09a9ae5c250e1ab9c42ba9edccc476263", [:mix], [], "hexpm"}, "hackney": {:hex, :hackney, "1.8.6", "21a725db3569b3fb11a6af17d5c5f654052ce9624219f1317e8639183de4a423", [:rebar3], [{:certifi, "1.2.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.0.2", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "httpoison": {:hex, :httpoison, "0.11.2", "9e59f17a473ef6948f63c51db07320477bad8ba88cf1df60a3eee01150306665", [:mix], [{:hackney, "~> 1.8.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, + "hypermock": {:git, "https://github.com/croesnick/hypermock", "0d7996b88953aa263844d479409bde82e6b8c5d7", []}, "idna": {:hex, :idna, "5.0.2", "ac203208ada855d95dc591a764b6e87259cb0e2a364218f215ad662daa8cd6b4", [:rebar3], [{:unicode_util_compat, "0.2.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"}, "inflex": {:hex, :inflex, "1.8.1", "9fa9684ff1a872eab7415c0be500cc1b7782f28da6ed75423081e75f92831b1c", [:mix], [], "hexpm"}, "json_web_token": {:hex, :json_web_token, "0.2.8", "c79d4c36cfd6f205be7099713e67d6057bde64ee9c64363f9001e3de86de703c", [:mix], [{:poison, "~> 3.1", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, + "jsx": {:hex, :jsx, "2.8.2", "7acc7d785b5abe8a6e9adbde926a24e481f29956dd8b4df49e3e4e7bcc92a018", [:mix, :rebar3], [], "hexpm"}, "kafka_ex": {:hex, :kafka_ex, "0.6.5", "01c160cbfbe67f6dd60da446b7e7e43f3e51ceb943478f9f7ac425ac0f925b18", [:mix], [], "hexpm"}, "maru": {:hex, :maru, "0.11.4", "35f4a0ab0eab5247f6b5b74bba30fcb63c575ea3157afb3dac0b6804b94a2a86", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 1.5 or ~> 2.0 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "meck": {:hex, :meck, "0.8.4", "59ca1cd971372aa223138efcf9b29475bde299e1953046a0c727184790ab1520", [:make, :rebar], [], "hexpm"}, @@ -27,12 +33,14 @@ "mime": {:hex, :mime, "1.1.0", "01c1d6f4083d8aa5c7b8c246ade95139620ef8effb009edde934e0ec3b28090a", [:mix], [], "hexpm"}, "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], [], "hexpm"}, "mochijson3": {:git, "https://github.com/tophitpoker/mochijson3.git", "1a1c913ac80bb45d3de5fbd74d21e96c45e9e844", [branch: "master"]}, + "mock": {:hex, :mock, "0.2.1", "bfdba786903e77f9c18772dee472d020ceb8ef000783e737725a4c8f54ad28ec", [:mix], [{:meck, "~> 0.8.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, "plug": {:hex, :plug, "1.3.5", "7503bfcd7091df2a9761ef8cecea666d1f2cc454cbbaf0afa0b6e259203b7031", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1", [hex: :cowboy, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], [], "hexpm"}, "postgrex": {:hex, :postgrex, "0.13.3", "c277cfb2a9c5034d445a722494c13359e361d344ef6f25d604c2353185682bfc", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 1.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm"}, "ranch": {:hex, :ranch, "1.3.2", "e4965a144dc9fbe70e5c077c65e73c57165416a901bd02ea899cfd95aa890986", [:rebar3], [], "hexpm"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], [], "hexpm"}, + "stubr": {:hex, :stubr, "1.5.0", "68752582482a4af06b88bd718d652026ac4216508e7097c4aae7b924c4b9bc4d", [:mix], [], "hexpm"}, "timex": {:hex, :timex, "3.1.15", "94abaec8fef2436ced4d0e1b4ed50c8eaa5fb9138fc0699946ddee7abf5aaff2", [:mix], [{:combine, "~> 0.7", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "timex_ecto": {:hex, :timex_ecto, "3.1.1", "37d54f6879d96a6789bb497296531cfb853631de78e152969d95cff03c1368dd", [:mix], [{:ecto, "~> 2.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:timex, "~> 3.0", [hex: :timex, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.12", "1c17b68692c6ba5b6ab15db3d64cc8baa0f182043d5ae9d4b6d35d70af76f67b", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/geronimo/spec/factories/ecto_factory.ex b/geronimo/spec/factories/ecto_factory.ex index 5caee18fee..8c7d14fd48 100644 --- a/geronimo/spec/factories/ecto_factory.ex +++ b/geronimo/spec/factories/ecto_factory.ex @@ -7,9 +7,10 @@ defmodule Geronimo.EctoFactory do def content_type_factory do %ContentType{ name: "FooBar", - schema: %{foo: %{type: ["string"], requred: true}, - bar: %{type: ["string"], requred: false}, - baz: %{type: ["array", []], requred: true}}, + schema: %{"author" => %{"required" => true, "type" => ["string"]}, + "body" => %{"required" => true, "type" => ["string"], "widget" => "richText"}, + "tags" => %{"required" => false, "type" => ["array", []]}, + "title" => %{"required" => true, "type" => ["string"]}}, scope: "1", created_by: 1 } @@ -18,10 +19,11 @@ defmodule Geronimo.EctoFactory do def valid_entity_factory do %Entity{ kind: "Foo", - content: %{foo: "foo", bar: "bar", baz: [1, 2, 3]}, + content: %{"author" => "foo", "body" => "bar", "tags" => [1, 2, 3], "title" => "quuux"}, schema_version: "", content_type_id: 1, created_by: 1, + storefront: "foo", scope: "1" } end @@ -33,6 +35,7 @@ defmodule Geronimo.EctoFactory do schema_version: "", content_type_id: 1, created_by: 1, + storefront: "foo", scope: "1" } end diff --git a/geronimo/spec/fixture/vcr_cassettes/schema_registry/err.json b/geronimo/spec/fixture/vcr_cassettes/schema_registry/err.json new file mode 100644 index 0000000000..88d6b198ff --- /dev/null +++ b/geronimo/spec/fixture/vcr_cassettes/schema_registry/err.json @@ -0,0 +1,23 @@ +[ + { + "request": { + "body": "{\"schema\":\"wrong!\"}", + "headers": [], + "method": "post", + "options": [], + "request_body": "", + "url": "http://127.0.0.1:8081/subjects/entities-key/versions" + }, + "response": { + "body": "{\"error_code\":42201,\"message\":\"Input schema is an invalid Avro schema\"}", + "headers": { + "Date": "Fri, 23 Jun 2017 05:38:25 GMT", + "Content-Type": "application/vnd.schemaregistry.v1+json", + "Content-Length": "71", + "Server": "Jetty(9.2.12.v20150709)" + }, + "status_code": 422, + "type": "ok" + } + } +] \ No newline at end of file diff --git a/geronimo/spec/fixture/vcr_cassettes/schema_registry/fail1.json b/geronimo/spec/fixture/vcr_cassettes/schema_registry/fail1.json new file mode 100644 index 0000000000..87164b585a --- /dev/null +++ b/geronimo/spec/fixture/vcr_cassettes/schema_registry/fail1.json @@ -0,0 +1,18 @@ +[ + { + "request": { + "body": "", + "headers": [], + "method": "get", + "options": [], + "request_body": "", + "url": "http://127.0.0.1:8081/subjects/entities-key/versions/1" + }, + "response": { + "body": "econnrefused", + "headers": [], + "status_code": null, + "type": "error" + } + } +] \ No newline at end of file diff --git a/geronimo/spec/fixture/vcr_cassettes/schema_registry/fail2.json b/geronimo/spec/fixture/vcr_cassettes/schema_registry/fail2.json new file mode 100644 index 0000000000..be592b9485 --- /dev/null +++ b/geronimo/spec/fixture/vcr_cassettes/schema_registry/fail2.json @@ -0,0 +1,18 @@ +[ + { + "request": { + "body": "{\"schema\":{\"foo\":\"bar\"}}", + "headers": [], + "method": "post", + "options": [], + "request_body": "", + "url": "http://127.0.0.1:8081/subjects/entities-key/versions" + }, + "response": { + "body": "econnrefused", + "headers": [], + "status_code": null, + "type": "error" + } + } +] \ No newline at end of file diff --git a/geronimo/spec/fixture/vcr_cassettes/schema_registry/not_found.json b/geronimo/spec/fixture/vcr_cassettes/schema_registry/not_found.json new file mode 100644 index 0000000000..a6db905cc9 --- /dev/null +++ b/geronimo/spec/fixture/vcr_cassettes/schema_registry/not_found.json @@ -0,0 +1,23 @@ +[ + { + "request": { + "body": "", + "headers": [], + "method": "get", + "options": [], + "request_body": "", + "url": "http://127.0.0.1:8081/subjects/entities-key/versions/112121212" + }, + "response": { + "body": "{\"error_code\":40402,\"message\":\"Version not found.\"}", + "headers": { + "Date": "Fri, 23 Jun 2017 05:38:24 GMT", + "Content-Type": "application/vnd.schemaregistry.v1+json", + "Content-Length": "51", + "Server": "Jetty(9.2.12.v20150709)" + }, + "status_code": 404, + "type": "ok" + } + } +] \ No newline at end of file diff --git a/geronimo/spec/fixture/vcr_cassettes/schema_registry/store_success.json b/geronimo/spec/fixture/vcr_cassettes/schema_registry/store_success.json new file mode 100644 index 0000000000..89b343a345 --- /dev/null +++ b/geronimo/spec/fixture/vcr_cassettes/schema_registry/store_success.json @@ -0,0 +1,23 @@ +[ + { + "request": { + "body": "{\"schema\":\"{\\\"type\\\":\\\"record\\\",\\\"namespace\\\":\\\"com.foxcommerce.geronimo\\\",\\\"name\\\":\\\"entities_pkey\\\",\\\"fields\\\":[{\\\"type\\\":[\\\"null\\\",\\\"int\\\"],\\\"name\\\":\\\"id\\\"}]}\"}", + "headers": [], + "method": "post", + "options": [], + "request_body": "", + "url": "http://127.0.0.1:8081/subjects/entities-key/versions" + }, + "response": { + "body": "{\"id\":140}", + "headers": { + "Date": "Fri, 23 Jun 2017 05:38:25 GMT", + "Content-Type": "application/vnd.schemaregistry.v1+json", + "Content-Length": "10", + "Server": "Jetty(9.2.12.v20150709)" + }, + "status_code": 200, + "type": "ok" + } + } +] \ No newline at end of file diff --git a/geronimo/spec/fixture/vcr_cassettes/schema_registry/success_get.json b/geronimo/spec/fixture/vcr_cassettes/schema_registry/success_get.json new file mode 100644 index 0000000000..717d60ca81 --- /dev/null +++ b/geronimo/spec/fixture/vcr_cassettes/schema_registry/success_get.json @@ -0,0 +1,23 @@ +[ + { + "request": { + "body": "", + "headers": [], + "method": "get", + "options": [], + "request_body": "", + "url": "http://127.0.0.1:8081/subjects/entities-key/versions/1" + }, + "response": { + "body": "{\"subject\":\"entities-key\",\"version\":1,\"id\":140,\"schema\":\"{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"entities_pkey\\\",\\\"namespace\\\":\\\"com.foxcommerce.geronimo\\\",\\\"fields\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":[\\\"null\\\",\\\"int\\\"]}]}\"}", + "headers": { + "Date": "Fri, 23 Jun 2017 05:33:17 GMT", + "Content-Type": "application/vnd.schemaregistry.v1+json", + "Content-Length": "209", + "Server": "Jetty(9.2.12.v20150709)" + }, + "status_code": 200, + "type": "ok" + } + } +] \ No newline at end of file diff --git a/geronimo/spec/lib/geronimo/kafka/schema_registry_client_spec.exs b/geronimo/spec/lib/geronimo/kafka/schema_registry_client_spec.exs new file mode 100644 index 0000000000..377ce1002d --- /dev/null +++ b/geronimo/spec/lib/geronimo/kafka/schema_registry_client_spec.exs @@ -0,0 +1,68 @@ +defmodule SchemaRegistryClientSpec do + use ESpec + use ExVCR.Mock, adapter: ExVCR.Adapter.Hackney + + before do + ExVCR.Config.cassette_library_dir("spec/fixture/vcr_cassettes") + :ok + end + + describe "get_schema" do + context "when response is successful" do + it "should return successful response" do + use_cassette "schema_registry/success_get" do + resp = {:ok, %{id: 140, schema: "{\"type\":\"record\",\"name\":\"entities_pkey\",\"namespace\":\"com.foxcommerce.geronimo\",\"fields\":[{\"name\":\"id\",\"type\":[\"null\",\"int\"]}]}", subject: "entities-key", version: 1}} + expect(Geronimo.Kafka.SchemaRegistryClient.get_schema("entities-key", 1)).to eq(resp) + end + end + end + + + context "when schema is not found" do + it "should return not_found response" do + use_cassette "schema_registry/not_found" do + resp = {:error, %{error_code: 40402, message: "Version not found."}} + expect(Geronimo.Kafka.SchemaRegistryClient.get_schema("entities-key", 112121212)).to eq(resp) + end + end + end + + context "when request failed" do + it "should return error response" do + use_cassette "schema_registry/fail1" do + resp = {:fail, "econnrefused"} + expect(Geronimo.Kafka.SchemaRegistryClient.get_schema("entities-key", 1)).to eq(resp) + end + end + end + end + + describe "store_schema" do + context "when response is successful" do + it "should return successful response" do + use_cassette "schema_registry/store_success" do + resp = {:ok, %{id: 140}} + expect(Geronimo.Kafka.SchemaRegistryClient.store_schema("entities-key", Geronimo.Entity.avro_schema_key)).to eq({:ok, %{id: 140}}) + end + end + end + + context "when schema can not be processed" do + it "should return error response" do + use_cassette "schema_registry/err" do + resp = {:error, %{error_code: 42201, message: "Input schema is an invalid Avro schema"}} + expect(Geronimo.Kafka.SchemaRegistryClient.store_schema("entities-key", "wrong!")).to eq(resp) + end + end + end + + context "when request failed" do + it "should return noerrort_found response" do + use_cassette "schema_registry/fail2" do + resp = {:fail, "econnrefused"} + expect(Geronimo.Kafka.SchemaRegistryClient.store_schema("entities-key", %{foo: "bar"})).to eq(resp) + end + end + end + end +end diff --git a/geronimo/spec/lib/geronimo/models/content_type_spec.exs b/geronimo/spec/lib/geronimo/models/content_type_spec.exs new file mode 100644 index 0000000000..b36cab50f7 --- /dev/null +++ b/geronimo/spec/lib/geronimo/models/content_type_spec.exs @@ -0,0 +1,30 @@ +defmodule ContentTypeSpec do + use ESpec + import Geronimo.Factory + + describe "to_avro" do + context "when record id passed" do + it "should return avro schema" do + content_type = insert(:content_type) + schema = Geronimo.ContentType.to_avro(content_type.id) + expect(schema["name"]).to eq(content_type.name) + expect(schema["created_by"]).to eq(content_type.created_by) + expect(schema["scope"]).to eq(content_type.scope) + expect(schema["schema"]).to eq(Poison.encode!(content_type.schema)) + expect(schema["id"]).to eq(content_type.id) + end + end + + context "when record is passed" do + it "should return avro schema" do + content_type = insert(:content_type) + schema = Geronimo.ContentType.to_avro(content_type) + expect(schema["name"]).to eq(content_type.name) + expect(schema["created_by"]).to eq(content_type.created_by) + expect(schema["scope"]).to eq(content_type.scope) + expect(schema["schema"]).to eq(Poison.encode!(content_type.schema)) + expect(schema["id"]).to eq(content_type.id) + end + end + end +end diff --git a/geronimo/spec/lib/geronimo/utils/avro_spec.exs b/geronimo/spec/lib/geronimo/utils/avro_spec.exs index 0ed1137ed5..9c9e9e127c 100644 --- a/geronimo/spec/lib/geronimo/utils/avro_spec.exs +++ b/geronimo/spec/lib/geronimo/utils/avro_spec.exs @@ -2,7 +2,7 @@ defmodule AvroSpec do use ESpec import Geronimo.Factory - describe "avro_encoding" do + describe "avro_encode" do it "encodes a model" do content_type = insert(:content_type) entity_attrs = %{schema_version: content_type.updated_at, content_type_id: content_type.id} @@ -17,7 +17,9 @@ defmodule AvroSpec do expect Geronimo.ContentType.avro_encode!(content_type) |> to(be_binary) expect Geronimo.Entity.avro_encode!(entity) |> to(be_binary) end + end # avro_encode + describe "avro_decode" do it "decodes a model" do content_type = insert(:content_type) entity_attrs = %{schema_version: content_type.updated_at, content_type_id: content_type.id} @@ -28,37 +30,70 @@ defmodule AvroSpec do expect Geronimo.ContentType.avro_decode!(ct_binary) |> to(be_map) expect Geronimo.Entity.avro_decode!(e_binary) |> to(be_map) end + end # avro_decode - it "returns avro schema for a model as a map" do - ct_schema = %{fields: [%{"name" => "created_by", "type" => ["int", "null"]}, - %{"name" => "id", "type" => ["int", "null"]}, - %{"name" => "inserted_at", "type" => ["string", "null"]}, - %{"name" => "name", "type" => ["string", "null"]}, - %{"name" => "schema", "type" => ["string", "null"]}, - %{"name" => "scope", "type" => ["string", "null"]}, - %{"name" => "updated_at", "type" => ["string", "null"]}], - name: "content_type", namespace: "com.foxcommerce.geronimo", type: "record"} - e_schema = %{fields: [%{"name" => "content", "type" => ["string", "null"]}, - %{"name" => "content_type_id", "type" => ["int", "null"]}, - %{"name" => "created_by", "type" => ["int", "null"]}, - %{"name" => "id", "type" => ["int", "null"]}, - %{"name" => "inserted_at", "type" => ["string", "null"]}, - %{"name" => "kind", "type" => ["string", "null"]}, - %{"name" => "schema_version", "type" => ["string", "null"]}, - %{"name" => "scope", "type" => ["string", "null"]}, - %{"name" => "updated_at", "type" => ["string", "null"]}], name: "entity", - namespace: "com.foxcommerce.geronimo", type: "record"} + describe "avro_schema_value" do + context "when returns schema as a map" do + it "returns avro schema value for a model as a map" do + ct_schema = %{fields: [%{"name" => "created_by", "type" => ["int", "null"]}, + %{"name" => "id", "type" => ["int", "null"]}, + %{"name" => "inserted_at", "type" => ["string", "null"]}, + %{"name" => "name", "type" => ["string", "null"]}, + %{"name" => "schema", "type" => ["string", "null"]}, + %{"name" => "scope", "type" => ["string", "null"]}, + %{"name" => "updated_at", "type" => ["string", "null"]}], + name: "content_types", namespace: "com.foxcommerce.geronimo", type: "record"} - expect(Geronimo.ContentType.avro_schema(false)).to eq(ct_schema) - expect(Geronimo.Entity.avro_schema(false)).to eq(e_schema) - end + e_schema = %{fields: [%{"name" => "content", "type" => ["string", "null"]}, + %{"name" => "content_type_id", "type" => ["int", "null"]}, + %{"name" => "created_by", "type" => ["int", "null"]}, + %{"name" => "id", "type" => ["int", "null"]}, + %{"name" => "inserted_at", "type" => ["string", "null"]}, + %{"name" => "kind", "type" => ["string", "null"]}, + %{"name" => "schema_version", "type" => ["string", "null"]}, + %{"name" => "scope", "type" => ["string", "null"]}, + %{"name" => "storefront", "type" => ["string", "null"]}, + %{"name" => "updated_at", "type" => ["string", "null"]}], name: "entities", + namespace: "com.foxcommerce.geronimo", type: "record"} - it "returns avro schema as a string" do - ct_schema_st = "{\"type\":\"record\",\"namespace\":\"com.foxcommerce.geronimo\",\"name\":\"content_type\",\"fields\":[{\"type\":[\"int\",\"null\"],\"name\":\"created_by\"},{\"type\":[\"int\",\"null\"],\"name\":\"id\"},{\"type\":[\"string\",\"null\"],\"name\":\"inserted_at\"},{\"type\":[\"string\",\"null\"],\"name\":\"name\"},{\"type\":[\"string\",\"null\"],\"name\":\"schema\"},{\"type\":[\"string\",\"null\"],\"name\":\"scope\"},{\"type\":[\"string\",\"null\"],\"name\":\"updated_at\"}]}" - e_schema_st = "{\"type\":\"record\",\"namespace\":\"com.foxcommerce.geronimo\",\"name\":\"entity\",\"fields\":[{\"type\":[\"string\",\"null\"],\"name\":\"content\"},{\"type\":[\"int\",\"null\"],\"name\":\"content_type_id\"},{\"type\":[\"int\",\"null\"],\"name\":\"created_by\"},{\"type\":[\"int\",\"null\"],\"name\":\"id\"},{\"type\":[\"string\",\"null\"],\"name\":\"inserted_at\"},{\"type\":[\"string\",\"null\"],\"name\":\"kind\"},{\"type\":[\"string\",\"null\"],\"name\":\"schema_version\"},{\"type\":[\"string\",\"null\"],\"name\":\"scope\"},{\"type\":[\"string\",\"null\"],\"name\":\"updated_at\"}]}" + expect(Geronimo.ContentType.avro_schema_value(false)).to eq(ct_schema) + expect(Geronimo.Entity.avro_schema_value(false)).to eq(e_schema) + end + end # when returns schema as a map - expect(Geronimo.ContentType.avro_schema).to eq(ct_schema_st) - expect(Geronimo.Entity.avro_schema).to eq(e_schema_st) - end - end + context "when returns schema as a string" do + it "returns avro schema value as a string" do + ct_schema_st = "{\"type\":\"record\",\"namespace\":\"com.foxcommerce.geronimo\",\"name\":\"content_types\",\"fields\":[{\"type\":[\"int\",\"null\"],\"name\":\"created_by\"},{\"type\":[\"int\",\"null\"],\"name\":\"id\"},{\"type\":[\"string\",\"null\"],\"name\":\"inserted_at\"},{\"type\":[\"string\",\"null\"],\"name\":\"name\"},{\"type\":[\"string\",\"null\"],\"name\":\"schema\"},{\"type\":[\"string\",\"null\"],\"name\":\"scope\"},{\"type\":[\"string\",\"null\"],\"name\":\"updated_at\"}]}" + e_schema_st = "{\"type\":\"record\",\"namespace\":\"com.foxcommerce.geronimo\",\"name\":\"entities\",\"fields\":[{\"type\":[\"string\",\"null\"],\"name\":\"content\"},{\"type\":[\"int\",\"null\"],\"name\":\"content_type_id\"},{\"type\":[\"int\",\"null\"],\"name\":\"created_by\"},{\"type\":[\"int\",\"null\"],\"name\":\"id\"},{\"type\":[\"string\",\"null\"],\"name\":\"inserted_at\"},{\"type\":[\"string\",\"null\"],\"name\":\"kind\"},{\"type\":[\"string\",\"null\"],\"name\":\"schema_version\"},{\"type\":[\"string\",\"null\"],\"name\":\"scope\"},{\"type\":[\"string\",\"null\"],\"name\":\"storefront\"},{\"type\":[\"string\",\"null\"],\"name\":\"updated_at\"}]}" + + expect(Geronimo.ContentType.avro_schema_value).to eq(ct_schema_st) + expect(Geronimo.Entity.avro_schema_value).to eq(e_schema_st) + end + end # when returns schema as a string + end # avro_schema_value + + describe "avro_schema_key" do + context "when returns schema key as a map" do + it "returns avro key schema as a map" do + k_entity = %{fields: [%{"name" => "id", "type" => ["null", "int"]}], name: "entities_pkey", + namespace: "com.foxcommerce.geronimo", type: "record"} + k_content_type = %{fields: [%{"name" => "id", "type" => ["null", "int"]}], + name: "content_types_pkey", namespace: "com.foxcommerce.geronimo", + type: "record"} + + expect(Geronimo.ContentType.avro_schema_key(false)).to eq(k_content_type) + expect(Geronimo.Entity.avro_schema_key(false)).to eq(k_entity) + end + end # when returns schema key as a map + + context "when returns schema key as a string" do + it "should return avro key schema as a string" do + k_entity_st = "{\"type\":\"record\",\"namespace\":\"com.foxcommerce.geronimo\",\"name\":\"entities_pkey\",\"fields\":[{\"type\":[\"null\",\"int\"],\"name\":\"id\"}]}" + k_content_type_st = "{\"type\":\"record\",\"namespace\":\"com.foxcommerce.geronimo\",\"name\":\"content_types_pkey\",\"fields\":[{\"type\":[\"null\",\"int\"],\"name\":\"id\"}]}" + + expect(Geronimo.ContentType.avro_schema_key).to eq(k_content_type_st) + expect(Geronimo.Entity.avro_schema_key).to eq(k_entity_st) + end + end # when returns schema key as a string + end # avro_schema_key end \ No newline at end of file diff --git a/geronimo/spec/lib/geronimo/utils/crud_spec.exs b/geronimo/spec/lib/geronimo/utils/crud_spec.exs new file mode 100644 index 0000000000..18ba57c885 --- /dev/null +++ b/geronimo/spec/lib/geronimo/utils/crud_spec.exs @@ -0,0 +1,58 @@ +defmodule CrudSpec do + use ESpec + import Geronimo.Factory + + describe "get_all" do + context "when there are some records for scope" do + it "sgould return list" do + ct = insert(:content_type) + expect(Geronimo.ContentType.get_all("1")).to have_length(1) + end + end + + context "when no records for given scope" do + it "sgould return an empty list" do + ct = insert(:content_type) + expect(Geronimo.ContentType.get_all("99")).to have_length(0) + end + end + end # get_all + + describe "get" do + context "when content_type exists" do + it "should return content_type" do + ct = insert(:content_type) + {:ok, res} = Geronimo.ContentType.get(ct.id) + expect(res.name).to eq(ct.name) + expect(res.versions).to eq([]) + expect(res.created_by).to eq(ct.created_by) + expect(res.scope).to eq(ct.scope) + end + end + + context "when content_type doesn't exists" do + it "should return content_type" do + ct = insert(:content_type) + expect(Geronimo.ContentType.get("99")).to eq({:error, "Not found"}) + end + end + end # get + + describe "get_versions" do + context "when no versions present" do + it "should return empty list" do + ct = insert(:content_type) + expect Geronimo.ContentType.get_versions(ct.id) |> to(eq []) + end + end + + context "when versions are present" do + it "should return versions list" do + ct = insert(:content_type) + ch = Geronimo.ContentType.changeset(ct, %{name: "BazQuux"}) + Geronimo.Repo.update(ch) + expect(Geronimo.ContentType.get_versions(ct.id)).to have_length(1) + end + end + end +end diff --git a/geronimo/spec/lib/geronimo/validation_spec.exs b/geronimo/spec/lib/geronimo/validation_spec.exs new file mode 100644 index 0000000000..b9e83f9d61 --- /dev/null +++ b/geronimo/spec/lib/geronimo/validation_spec.exs @@ -0,0 +1,108 @@ +defmodule ValidationSpec do + use ESpec + import Geronimo.Factory + + describe "validate!" do + context "when valid params are passed" do + it "should return {:ok, valid_data}" do + schema = params_for(:content_type).schema + data = params_for(:valid_entity).content + expect Geronimo.Validation.validate!(data, schema) + |> to(eq {:ok, %{author: "foo", body: "bar", tags: [1, 2, 3], title: "quuux"}}) + end + end # valid params + + context "when params are invalid" do + it "should return {:error, valid_data}" do + schema = params_for(:content_type).schema + expect Geronimo.Validation.validate!(%{}, schema) + |> to(eq [{:error, :author, :presence, "must be present"}, + {:error, :body, :presence, "must be present"}, + {:error, :title, :presence, "must be present"}]) + end + end + + end + + describe "validate integer" do + let :schema, do: %{"amount" => %{"required" => true, "type" => ["integer"]}} + + context "when integer is passed" do + it "should return :ok" do + expect Geronimo.Validation.validate!(%{"amount" => 123}, schema) + |> to(eq {:ok, %{amount: 123}}) + end + end # :ok + + context "when string is passed" do + it "should return an error" do + expect Geronimo.Validation.validate!(%{"amount" => "foo"}, schema) + |> to(eq [{:error, :amount, :format, "only digits accepted"}]) + end + end # :error + end # integers + + describe "validate decimal" do + let :schema, do: %{"amount" => %{"required" => true, "type" => ["float"]}} + + context "when decimal is passed" do + it "should return :ok" do + expect Geronimo.Validation.validate!(%{"amount" => 12.3}, schema) + |> to(eq {:ok, %{amount: 12.3}}) + end + end # :ok + + context "when string is passed" do + it "should return an error" do + expect Geronimo.Validation.validate!(%{"amount" => "foo"}, schema) + |> to(eq [{:error, :amount, :format, "should be like 123.45"}]) + end + end # :error + end # decimals + + describe "validate arrays" do + let :schema, do: %{"tags" => %{"required" => true, "type" => ["array"]}} + + context "when array is passed" do + it "should return :ok" do + expect Geronimo.Validation.validate!(%{"tags" => [1, 2]}, schema) + |> to(eq {:ok, %{tags: [1, 2]}}) + end + + it "should return :ok on empty arrays" do + expect Geronimo.Validation.validate!(%{"tags" => []}, %{"tags" => %{"required" => false, "type" => ["array"]}}) + |> to(eq {:ok, %{tags: []}}) + end + end # :ok + + context "when string is passed" do + it "should return an error" do + expect Geronimo.Validation.validate!(%{"tags" => "foo"}, schema) + |> to(eq [{:error, :tags, :by, "must be valid"}]) + end + + it "should return :error on wrong format but not required" do + expect Geronimo.Validation.validate!(%{"tags" => "foo"}, %{"tags" => %{"required" => false, "type" => ["array"]}}) + |> to(eq [{:error, :tags, :by, "must be valid"}]) + end + end # :error + end # arrays + + describe "validate date" do + let :schema, do: %{"date" => %{"required" => true, "type" => ["date"]}} + + context "when date is passed" do + it "should return :ok" do + expect Geronimo.Validation.validate!(%{"date" => "1970-01-01T12:12:12.123456Z"}, schema) + |> to(eq {:ok, %{date: "1970-01-01T12:12:12.123456Z"}}) + end + end # :ok + + context "when string is passed" do + it "should return an error" do + expect Geronimo.Validation.validate!(%{"date" => "foo"}, schema) + |> to(eq [{:error, :date, :by, "should be like 1970-01-01T00:00:00.0Z"}]) + end + end # :error + end # arrays +end diff --git a/geronimo/sql/V1_004__add_storefront_to_entities.sql b/geronimo/sql/V1_004__add_storefront_to_entities.sql new file mode 100644 index 0000000000..04861d1dd9 --- /dev/null +++ b/geronimo/sql/V1_004__add_storefront_to_entities.sql @@ -0,0 +1 @@ +alter table entities add column storefront varchar; \ No newline at end of file diff --git a/green-river/src/main/resources/application.conf b/green-river/src/main/resources/application.conf index d41f819102..dba012270a 100644 --- a/green-river/src/main/resources/application.conf +++ b/green-river/src/main/resources/application.conf @@ -3,7 +3,8 @@ kafka.indices { "countries", "regions", "products_catalog_view", - "product_reviews_search_view" + "product_reviews_search_view", + "geronimo_entities" ] admin = [ @@ -34,20 +35,22 @@ kafka.scoped.indices { "inventory_transactions_search_view", "taxonomies_search_view", "taxons_search_view", - "scoped_activity_trails" + "scoped_activity_trails", + "geronimo_entities", + "geronimo_content_types" ] } default { - avro.schemaRegistryUrl = "http://127.0.0.1:8081" - elastic.host = "elasticsearch://127.0.0.1:9300" - elastic.cluster = "hawk" + avro.schemaRegistryUrl = "http://10.240.0.17:8081" + elastic.host = "elasticsearch://10.240.0.17:9300" + elastic.cluster = "elasticsearch" elastic.index = "phoenix" elastic.setup = "false" - kafka.broker = "127.0.0.1:9092" + kafka.broker = "10.240.0.17:9092" kafka.groupId = "mox" activity.kafka.topic = "scoped_activities" - activity.phoenix.url = "http://127.0.0.1:9090/v1" + activity.phoenix.url = "http://10.240.0.17:9090/v1" activity.phoenix.user = "api@foxcommerce.com" activity.phoenix.pass = "api$pass7!" activity.phoenix.org = "tenant" diff --git a/green-river/src/main/scala/consumer/AvroProcessor.scala b/green-river/src/main/scala/consumer/AvroProcessor.scala index fa33df9913..2f4d767fe6 100644 --- a/green-river/src/main/scala/consumer/AvroProcessor.scala +++ b/green-river/src/main/scala/consumer/AvroProcessor.scala @@ -130,6 +130,7 @@ class AvroProcessor(schemaRegistryUrl: String, processor: JsonProcessor)(implici def process(offset: Long, topic: String, key: Array[Byte], message: Array[Byte]): Future[Unit] = try { + Console.err.println(s"message: ${new String(message)}") val keyJson = if (key == null || key.isEmpty) { diff --git a/green-river/src/main/scala/consumer/Workers.scala b/green-river/src/main/scala/consumer/Workers.scala index b5026c1c3d..87c4e16f79 100644 --- a/green-river/src/main/scala/consumer/Workers.scala +++ b/green-river/src/main/scala/consumer/Workers.scala @@ -198,6 +198,9 @@ object Workers { "store_credits_search_view" → StoreCreditsSearchView(), "scoped_activity_trails" → ActivityConnectionTransformer(connectionInfo), "taxonomies_search_view" → TaxonomiesSearchView(), - "taxons_search_view" → TaxonsSearchView() + "taxons_search_view" → TaxonsSearchView(), + "geronimo_entities" → EntitiesAdminView(), + "geronimo_entities" → EntitiesPublicView(), + "geronimo_conten_type" → ContentTypesView() ) } diff --git a/green-river/src/main/scala/consumer/elastic/mappings/EntitiesPublicView.scala b/green-river/src/main/scala/consumer/elastic/mappings/EntitiesPublicView.scala new file mode 100644 index 0000000000..2295b6542a --- /dev/null +++ b/green-river/src/main/scala/consumer/elastic/mappings/EntitiesPublicView.scala @@ -0,0 +1,18 @@ +package consumer.elastic.mappings + +import com.sksamuel.elastic4s.ElasticDsl._ +import com.sksamuel.elastic4s.ElasticDsl.{mapping ⇒ esMapping} +import com.sksamuel.elastic4s.mappings.FieldType._ + +import consumer.aliases._ +import consumer.elastic.AvroTransformer + +final case class EntitiesPublicView()(implicit ec: EC) extends AvroTransformer { + def topic() = "geronimo_entities" + def mapping() = esMapping(topic()).fields( + field("id", LongType), + field("kind", StringType).index("not_analyzed"), + field("content", StringType).index("not_analyzed"), + field("storefront", StringType).index("not_analyzed") + ) +} diff --git a/green-river/src/main/scala/consumer/elastic/mappings/admin/ContentTypesView.scala b/green-river/src/main/scala/consumer/elastic/mappings/admin/ContentTypesView.scala new file mode 100644 index 0000000000..2d1dc8b840 --- /dev/null +++ b/green-river/src/main/scala/consumer/elastic/mappings/admin/ContentTypesView.scala @@ -0,0 +1,21 @@ +package consumer.elastic.mappings.admin + +import com.sksamuel.elastic4s.ElasticDsl.{mapping ⇒ esMapping, _} +import com.sksamuel.elastic4s.mappings.FieldType._ +import consumer.aliases._ +import consumer.elastic.AvroTransformer +import consumer.elastic.mappings.dateFormat + +final case class ContentTypesView()(implicit ec: EC) extends AvroTransformer { + def topic() = "geronimo_content_types" + def mapping() = esMapping(topic()).fields( + field("id", LongType), + field("name", StringType).index("not_analyzed"), + field("schema", StringType).index("not_analyzed"), + field("scope", StringType).index("not_analyzed"), + field("created_by", IntegerType), + field("inserted_at", DateType).format(dateFormat), + field("updated_at", DateType).format(dateFormat), + field("scope", StringType).index("not_analyzed") + ) +} diff --git a/green-river/src/main/scala/consumer/elastic/mappings/admin/EntitiesAdminView.scala b/green-river/src/main/scala/consumer/elastic/mappings/admin/EntitiesAdminView.scala new file mode 100644 index 0000000000..a0f91e7f01 --- /dev/null +++ b/green-river/src/main/scala/consumer/elastic/mappings/admin/EntitiesAdminView.scala @@ -0,0 +1,23 @@ +package consumer.elastic.mappings.admin + +import com.sksamuel.elastic4s.ElasticDsl.{mapping ⇒ esMapping, _} +import com.sksamuel.elastic4s.mappings.FieldType._ +import consumer.aliases._ +import consumer.elastic.AvroTransformer +import consumer.elastic.mappings.dateFormat + +final case class EntitiesAdminView()(implicit ec: EC) extends AvroTransformer { + def topic() = "geronimo_entities" + def mapping() = esMapping(topic()).fields( + field("id", LongType), + field("kind", StringType).index("not_analyzed"), + field("content", StringType).index("not_analyzed"), + field("storefront", StringType).index("not_analyzed"), + field("schema_version", DateType).format(dateFormat), + field("content_type_id", IntegerType), + field("created_by", IntegerType), + field("scope", StringType).index("not_analyzed"), + field("inserted_at", DateType).format(dateFormat), + field("updated_at", DateType).format(dateFormat) + ) +}