From 508084dcad7387c5bd7ff30101b3ae86fa6f55b1 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Mon, 30 Mar 2020 18:22:40 +0700 Subject: [PATCH 1/8] Create keyword from the parse csv --- .../controllers/upload_controller.ex | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/lib/google_crawler_web/controllers/upload_controller.ex b/lib/google_crawler_web/controllers/upload_controller.ex index 02172d8..10e09b4 100644 --- a/lib/google_crawler_web/controllers/upload_controller.ex +++ b/lib/google_crawler_web/controllers/upload_controller.ex @@ -10,14 +10,46 @@ defmodule GoogleCrawlerWeb.UploadController do if changeset.valid? do file = get_change(changeset, :file, nil) - result = Search.parse_keywords_from_file!(file.path, file.content_type) - # TODO: Save these keywords and triggers the task to google search for each keyword - text(conn, result |> Enum.map(fn keyword -> List.first(keyword) end) |> Enum.join(", ")) + Search.parse_keywords_from_file!(file.path, file.content_type) + |> create_and_trigger_google_search + |> put_error_flash_for_failed_keywords(conn) + |> redirect(to: Routes.dashboard_path(conn, :index)) else conn |> put_flash(:error, gettext("Invalid file, please select again.")) |> redirect(to: Routes.dashboard_path(conn, :index)) end end + + # TODO: Trigger the scrapper background worker + defp create_and_trigger_google_search(csv_result) do + csv_result + |> Stream.map(fn keyword_row -> List.first(keyword_row) end) + |> Stream.map(fn keyword -> %{keyword: keyword} end) + |> Enum.map(&Search.create_keyword/1) + end + + defp put_error_flash_for_failed_keywords(create_result, conn) do + failed_keywords = failed_keywords(create_result) + + if length(failed_keywords) > 0 do + conn + |> put_flash(:error, + gettext("Some keywords could not be created: %{failed_keywords}", failed_keywords: Enum.join(failed_keywords, ",")) + ) + else + conn + end + end + + defp failed_keywords(create_result) do + create_result + |> Enum.filter(&match?({:error, _}, &1)) + |> Enum.map(fn error_tuple -> + error_tuple + |> elem(1) + |> get_change(:keyword, nil) + end) + end end From 0b46dee6526d7d2cdff871de3e8b00934baacf76 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Mon, 30 Mar 2020 18:56:09 +0700 Subject: [PATCH 2/8] Update the upload controller test --- lib/google_crawler_web/controllers/upload_controller.ex | 7 +++++-- .../controllers/upload_controller_test.exs | 9 +++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/google_crawler_web/controllers/upload_controller.ex b/lib/google_crawler_web/controllers/upload_controller.ex index 10e09b4..d4cb77c 100644 --- a/lib/google_crawler_web/controllers/upload_controller.ex +++ b/lib/google_crawler_web/controllers/upload_controller.ex @@ -35,8 +35,11 @@ defmodule GoogleCrawlerWeb.UploadController do if length(failed_keywords) > 0 do conn - |> put_flash(:error, - gettext("Some keywords could not be created: %{failed_keywords}", failed_keywords: Enum.join(failed_keywords, ",")) + |> put_flash( + :error, + gettext("Some keywords could not be created: %{failed_keywords}", + failed_keywords: Enum.join(failed_keywords, ",") + ) ) else conn diff --git a/test/google_crawler_web/controllers/upload_controller_test.exs b/test/google_crawler_web/controllers/upload_controller_test.exs index 4196b8a..2e3aeda 100644 --- a/test/google_crawler_web/controllers/upload_controller_test.exs +++ b/test/google_crawler_web/controllers/upload_controller_test.exs @@ -2,8 +2,11 @@ defmodule GoogleCrawlerWeb.UploadControllerTest do use GoogleCrawlerWeb.ConnCase alias GoogleCrawler.UserFactory + alias GoogleCrawler.Repo + alias GoogleCrawler.Search.Keyword - test "create/2 renders csv content as text if the keyword file is valid", %{conn: conn} do + test "create/2 creates keywords and redirects to the user dashboard if the keyword file is valid", + %{conn: conn} do user = UserFactory.create() upload_file = upload_file_fixture("keyword_files/valid_keyword.csv") @@ -11,7 +14,9 @@ defmodule GoogleCrawlerWeb.UploadControllerTest do build_authenticated_conn(user) |> post(Routes.upload_path(conn, :create), %{keyword_file: %{file: upload_file}}) - assert text_response(conn, 200) == "elixir, ruby, javascript" + keywords = Repo.all(Keyword) |> Enum.map(&Map.get(&1, :keyword)) + assert keywords == ["elixir", "ruby", "javascript"] + assert redirected_to(conn) == Routes.dashboard_path(conn, :index) end test "create/2 raises error if the file is failed to parse" do From 59b14fe5db934f9c6e9580b3723b08e254161685 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Mon, 30 Mar 2020 22:01:50 +0700 Subject: [PATCH 3/8] Associate the keywords with user --- lib/google_crawler/accounts/user.ex | 2 ++ lib/google_crawler/search.ex | 7 ++++--- lib/google_crawler/search/keyword.ex | 2 ++ lib/google_crawler_web/controllers/upload_controller.ex | 7 ++++--- .../20200330130512_add_user_id_to_keywords.exs | 9 +++++++++ test/factories/keyword_factory.ex | 5 +++-- test/google_crawler/search_test.exs | 7 +++++-- 7 files changed, 29 insertions(+), 10 deletions(-) create mode 100644 priv/repo/migrations/20200330130512_add_user_id_to_keywords.exs diff --git a/lib/google_crawler/accounts/user.ex b/lib/google_crawler/accounts/user.ex index 9538db0..f50ab45 100644 --- a/lib/google_crawler/accounts/user.ex +++ b/lib/google_crawler/accounts/user.ex @@ -10,6 +10,8 @@ defmodule GoogleCrawler.Accounts.User do field :password, :string, virtual: true field :password_confirmation, :string, virtual: true + has_many :keywords, GoogleCrawler.Search.Keyword + timestamps() end diff --git a/lib/google_crawler/search.ex b/lib/google_crawler/search.ex index 913aaf4..94aa70a 100644 --- a/lib/google_crawler/search.ex +++ b/lib/google_crawler/search.ex @@ -43,16 +43,17 @@ defmodule GoogleCrawler.Search do ## Examples - iex> create_keyword(%{field: value}) + iex> create_keyword(%User{}, %{field: value}) {:ok, %Keyword{}} - iex> create_keyword(%{field: bad_value}) + iex> create_keyword(%User{}, %{field: bad_value}) {:error, %Ecto.Changeset{}} """ - def create_keyword(attrs \\ %{}) do + def create_keyword(attrs \\ %{}, user) do %Keyword{} |> Keyword.changeset(attrs) + |> Ecto.Changeset.put_assoc(:user, user) |> Repo.insert() end diff --git a/lib/google_crawler/search/keyword.ex b/lib/google_crawler/search/keyword.ex index 50656c7..4efee14 100644 --- a/lib/google_crawler/search/keyword.ex +++ b/lib/google_crawler/search/keyword.ex @@ -5,6 +5,8 @@ defmodule GoogleCrawler.Search.Keyword do schema "keywords" do field :keyword, :string + belongs_to :user, GoogleCrawler.Accounts.User + timestamps() end diff --git a/lib/google_crawler_web/controllers/upload_controller.ex b/lib/google_crawler_web/controllers/upload_controller.ex index d4cb77c..e31fe7d 100644 --- a/lib/google_crawler_web/controllers/upload_controller.ex +++ b/lib/google_crawler_web/controllers/upload_controller.ex @@ -5,6 +5,7 @@ defmodule GoogleCrawlerWeb.UploadController do alias GoogleCrawler.Search alias GoogleCrawler.Search.KeywordFile + @spec create(atom | %{__struct__: atom}, map) :: Plug.Conn.t() def create(conn, %{"keyword_file" => keyword_file}) do changeset = KeywordFile.changeset(%KeywordFile{}, keyword_file) @@ -12,7 +13,7 @@ defmodule GoogleCrawlerWeb.UploadController do file = get_change(changeset, :file, nil) Search.parse_keywords_from_file!(file.path, file.content_type) - |> create_and_trigger_google_search + |> create_and_trigger_google_search(conn) |> put_error_flash_for_failed_keywords(conn) |> redirect(to: Routes.dashboard_path(conn, :index)) else @@ -23,11 +24,11 @@ defmodule GoogleCrawlerWeb.UploadController do end # TODO: Trigger the scrapper background worker - defp create_and_trigger_google_search(csv_result) do + defp create_and_trigger_google_search(csv_result, conn) do csv_result |> Stream.map(fn keyword_row -> List.first(keyword_row) end) |> Stream.map(fn keyword -> %{keyword: keyword} end) - |> Enum.map(&Search.create_keyword/1) + |> Enum.map(&Search.create_keyword(&1, conn.assigns.current_user)) end defp put_error_flash_for_failed_keywords(create_result, conn) do diff --git a/priv/repo/migrations/20200330130512_add_user_id_to_keywords.exs b/priv/repo/migrations/20200330130512_add_user_id_to_keywords.exs new file mode 100644 index 0000000..aaad0df --- /dev/null +++ b/priv/repo/migrations/20200330130512_add_user_id_to_keywords.exs @@ -0,0 +1,9 @@ +defmodule GoogleCrawler.Repo.Migrations.AddUserIdToKeywords do + use Ecto.Migration + + def change do + alter table(:keywords) do + add :user_id, references(:users) + end + end +end diff --git a/test/factories/keyword_factory.ex b/test/factories/keyword_factory.ex index 597c333..f8170fa 100644 --- a/test/factories/keyword_factory.ex +++ b/test/factories/keyword_factory.ex @@ -1,5 +1,6 @@ defmodule GoogleCrawler.KeywordFactory do alias GoogleCrawler.Search + alias GoogleCrawler.UserFactory def default_attrs do %{ @@ -11,10 +12,10 @@ defmodule GoogleCrawler.KeywordFactory do Enum.into(attrs, default_attrs()) end - def create(attrs \\ %{}) do + def create(attrs \\ %{}, user \\ UserFactory.create()) do keyword_attrs = build_attrs(attrs) - {:ok, keyword} = Search.create_keyword(keyword_attrs) + {:ok, keyword} = Search.create_keyword(keyword_attrs, user) keyword end diff --git a/test/google_crawler/search_test.exs b/test/google_crawler/search_test.exs index ceb3df5..257b8ae 100644 --- a/test/google_crawler/search_test.exs +++ b/test/google_crawler/search_test.exs @@ -4,6 +4,7 @@ defmodule GoogleCrawler.SearchTest do alias GoogleCrawler.Search alias GoogleCrawler.Search.Keyword alias GoogleCrawler.KeywordFactory + alias GoogleCrawler.UserFactory describe "keywords" do test "list_keywords/0 returns all keywords" do @@ -19,16 +20,18 @@ defmodule GoogleCrawler.SearchTest do end test "create_keyword/1 with valid data creates a keyword" do + user = UserFactory.create() keyword_attrs = KeywordFactory.build_attrs(%{keyword: "elixir"}) - assert {:ok, %Keyword{} = keyword} = Search.create_keyword(keyword_attrs) + assert {:ok, %Keyword{} = keyword} = Search.create_keyword(keyword_attrs, user) assert keyword.keyword == "elixir" end test "create_keyword/1 with invalid data returns error changeset" do + user = UserFactory.create() keyword_attrs = KeywordFactory.build_attrs(%{keyword: ""}) - assert {:error, %Ecto.Changeset{}} = Search.create_keyword(keyword_attrs) + assert {:error, %Ecto.Changeset{}} = Search.create_keyword(keyword_attrs, user) end end From a91b0b5afefd9591517a9a3d9ac12fffc8351a3d Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Tue, 31 Mar 2020 09:43:51 +0700 Subject: [PATCH 4/8] Display keyword list --- .../controllers/dashboard_controller.ex | 4 +++- .../templates/dashboard/index.html.eex | 14 ++++++++++++-- .../templates/keyword/_form.html.eex | 18 ++++++++---------- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/lib/google_crawler_web/controllers/dashboard_controller.ex b/lib/google_crawler_web/controllers/dashboard_controller.ex index e341dbb..7b59e0b 100644 --- a/lib/google_crawler_web/controllers/dashboard_controller.ex +++ b/lib/google_crawler_web/controllers/dashboard_controller.ex @@ -1,11 +1,13 @@ defmodule GoogleCrawlerWeb.DashboardController do use GoogleCrawlerWeb, :controller + alias GoogleCrawler.Search alias GoogleCrawler.Search.KeywordFile def index(conn, _params) do + keywords = Search.list_keywords() changeset = KeywordFile.changeset(%KeywordFile{}) - render(conn, "index.html", changeset: changeset) + render(conn, "index.html", keywords: keywords, changeset: changeset) end end diff --git a/lib/google_crawler_web/templates/dashboard/index.html.eex b/lib/google_crawler_web/templates/dashboard/index.html.eex index a3bf664..16d85d2 100644 --- a/lib/google_crawler_web/templates/dashboard/index.html.eex +++ b/lib/google_crawler_web/templates/dashboard/index.html.eex @@ -1,6 +1,16 @@ -<%= render GoogleCrawlerWeb.KeywordView, "_form.html", assigns %> +
+ <%= render GoogleCrawlerWeb.KeywordView, "_form.html", assigns %> +

<%= gettext("Keywords") %>

-

<%= gettext("You don't have any keywords.") %>

+ <%= if length(@keywords) == 0 do %> +

<%= gettext("You don't have any keywords.") %>

+ <% else %> +
    + <%= for keyword <- @keywords do %> +
  • <%= keyword.keyword %>
  • + <% end %> +
+ <% end %>
diff --git a/lib/google_crawler_web/templates/keyword/_form.html.eex b/lib/google_crawler_web/templates/keyword/_form.html.eex index d0f3471..cdae9e9 100644 --- a/lib/google_crawler_web/templates/keyword/_form.html.eex +++ b/lib/google_crawler_web/templates/keyword/_form.html.eex @@ -1,11 +1,9 @@ -
-

<%= gettext("Upload your keyword file (.csv)") %>

-

<%= gettext("📝 Please put one keyword per line") %>

- <%= form_for @changeset, Routes.upload_path(@conn, :create), [multipart: true], fn f -> %> - <%= label f, :file %> - <%= file_input f, :file, required: true %> - <%= error_tag f, :file %> +

<%= gettext("Upload your keyword file (.csv)") %>

+

<%= gettext("📝 Please put one keyword per line") %>

+<%= form_for @changeset, Routes.upload_path(@conn, :create), [multipart: true], fn f -> %> + <%= label f, :file %> + <%= file_input f, :file, required: true, accept: "text/csv" %> + <%= error_tag f, :file %> - <%= submit gettext("Upload") %> - <% end %> -
+ <%= submit gettext("Upload") %> +<% end %> From f4cf5214e20c0e47ddcc05f898b69d6dc730200f Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Tue, 31 Mar 2020 11:03:39 +0700 Subject: [PATCH 5/8] List the keywords that belongs to the user --- lib/google_crawler/search.ex | 10 ++++++---- .../controllers/dashboard_controller.ex | 2 +- test/google_crawler/search_test.exs | 11 ++++++++--- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/google_crawler/search.ex b/lib/google_crawler/search.ex index 94aa70a..1ae270b 100644 --- a/lib/google_crawler/search.ex +++ b/lib/google_crawler/search.ex @@ -10,16 +10,18 @@ defmodule GoogleCrawler.Search do alias GoogleCrawler.Search.KeywordFile @doc """ - Returns the list of keywords. + Returns the list of keywords belongs to the given user. ## Examples - iex> list_keywords() + iex> list_user_keywords(user) [%Keyword{}, ...] """ - def list_keywords do - Repo.all(Keyword) + def list_user_keywords(user) do + Keyword + |> where(user_id: ^user.id) + |> Repo.all() end @doc """ diff --git a/lib/google_crawler_web/controllers/dashboard_controller.ex b/lib/google_crawler_web/controllers/dashboard_controller.ex index 7b59e0b..c3e146d 100644 --- a/lib/google_crawler_web/controllers/dashboard_controller.ex +++ b/lib/google_crawler_web/controllers/dashboard_controller.ex @@ -5,7 +5,7 @@ defmodule GoogleCrawlerWeb.DashboardController do alias GoogleCrawler.Search.KeywordFile def index(conn, _params) do - keywords = Search.list_keywords() + keywords = Search.list_user_keywords(conn.assigns.current_user) changeset = KeywordFile.changeset(%KeywordFile{}) render(conn, "index.html", keywords: keywords, changeset: changeset) diff --git a/test/google_crawler/search_test.exs b/test/google_crawler/search_test.exs index 257b8ae..f4d7f80 100644 --- a/test/google_crawler/search_test.exs +++ b/test/google_crawler/search_test.exs @@ -7,10 +7,15 @@ defmodule GoogleCrawler.SearchTest do alias GoogleCrawler.UserFactory describe "keywords" do - test "list_keywords/0 returns all keywords" do - keyword = KeywordFactory.create() + test "list_user_keywords/0 returns all keywords" do + user1 = UserFactory.create() + user2 = UserFactory.create() + keyword1 = KeywordFactory.create(%{}, user1) + keyword2 = KeywordFactory.create(%{}, user2) + + user_keywords = Search.list_user_keywords(user1) - assert Search.list_keywords() |> Enum.map(&Map.get(&1, :keyword)) == [keyword.keyword] + assert user_keywords |> Enum.map(&Map.get(&1, :keyword)) == [keyword1.keyword] end test "get_keyword/1 returns the keyword with given id" do From b7fc208edd234bd49a500ca4c030e395b28adf20 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Tue, 31 Mar 2020 11:19:00 +0700 Subject: [PATCH 6/8] Update docs and reformat the error message --- lib/google_crawler/search.ex | 4 ++-- lib/google_crawler_web/controllers/upload_controller.ex | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/google_crawler/search.ex b/lib/google_crawler/search.ex index 1ae270b..e25691a 100644 --- a/lib/google_crawler/search.ex +++ b/lib/google_crawler/search.ex @@ -45,10 +45,10 @@ defmodule GoogleCrawler.Search do ## Examples - iex> create_keyword(%User{}, %{field: value}) + iex> create_keyword(%{field: value}, %User{}) {:ok, %Keyword{}} - iex> create_keyword(%User{}, %{field: bad_value}) + iex> create_keyword(%{field: bad_value}, %User{}) {:error, %Ecto.Changeset{}} """ diff --git a/lib/google_crawler_web/controllers/upload_controller.ex b/lib/google_crawler_web/controllers/upload_controller.ex index e31fe7d..e7ddf8b 100644 --- a/lib/google_crawler_web/controllers/upload_controller.ex +++ b/lib/google_crawler_web/controllers/upload_controller.ex @@ -5,7 +5,6 @@ defmodule GoogleCrawlerWeb.UploadController do alias GoogleCrawler.Search alias GoogleCrawler.Search.KeywordFile - @spec create(atom | %{__struct__: atom}, map) :: Plug.Conn.t() def create(conn, %{"keyword_file" => keyword_file}) do changeset = KeywordFile.changeset(%KeywordFile{}, keyword_file) @@ -39,7 +38,7 @@ defmodule GoogleCrawlerWeb.UploadController do |> put_flash( :error, gettext("Some keywords could not be created: %{failed_keywords}", - failed_keywords: Enum.join(failed_keywords, ",") + failed_keywords: Enum.join(failed_keywords, ", ") ) ) else From a44d3c4ed6b58dcfad404788a2dbb7d7b5bb6818 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Tue, 31 Mar 2020 12:32:52 +0700 Subject: [PATCH 7/8] Change put assoc to build assoc --- lib/google_crawler/search.ex | 3 +-- test/google_crawler/search_test.exs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/google_crawler/search.ex b/lib/google_crawler/search.ex index e25691a..9c261e0 100644 --- a/lib/google_crawler/search.ex +++ b/lib/google_crawler/search.ex @@ -53,9 +53,8 @@ defmodule GoogleCrawler.Search do """ def create_keyword(attrs \\ %{}, user) do - %Keyword{} + Ecto.build_assoc(user, :keywords) |> Keyword.changeset(attrs) - |> Ecto.Changeset.put_assoc(:user, user) |> Repo.insert() end diff --git a/test/google_crawler/search_test.exs b/test/google_crawler/search_test.exs index f4d7f80..e3f567b 100644 --- a/test/google_crawler/search_test.exs +++ b/test/google_crawler/search_test.exs @@ -11,7 +11,7 @@ defmodule GoogleCrawler.SearchTest do user1 = UserFactory.create() user2 = UserFactory.create() keyword1 = KeywordFactory.create(%{}, user1) - keyword2 = KeywordFactory.create(%{}, user2) + _keyword2 = KeywordFactory.create(%{}, user2) user_keywords = Search.list_user_keywords(user1) From 2832b0c9645f3a29579752bff4a6f8e2955efd0d Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Tue, 31 Mar 2020 14:39:53 +0700 Subject: [PATCH 8/8] Add association validation and test --- lib/google_crawler/search/keyword.ex | 4 +-- ...20200330130512_add_user_id_to_keywords.exs | 4 ++- test/google_crawler/search/keyword_test.exs | 26 +++++++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 test/google_crawler/search/keyword_test.exs diff --git a/lib/google_crawler/search/keyword.ex b/lib/google_crawler/search/keyword.ex index 4efee14..d87ca11 100644 --- a/lib/google_crawler/search/keyword.ex +++ b/lib/google_crawler/search/keyword.ex @@ -12,7 +12,7 @@ defmodule GoogleCrawler.Search.Keyword do def changeset(keyword, attrs \\ %{}) do keyword - |> cast(attrs, [:keyword]) - |> validate_required([:keyword]) + |> cast(attrs, [:keyword, :user_id]) + |> validate_required([:keyword, :user_id]) end end diff --git a/priv/repo/migrations/20200330130512_add_user_id_to_keywords.exs b/priv/repo/migrations/20200330130512_add_user_id_to_keywords.exs index aaad0df..672dc63 100644 --- a/priv/repo/migrations/20200330130512_add_user_id_to_keywords.exs +++ b/priv/repo/migrations/20200330130512_add_user_id_to_keywords.exs @@ -3,7 +3,9 @@ defmodule GoogleCrawler.Repo.Migrations.AddUserIdToKeywords do def change do alter table(:keywords) do - add :user_id, references(:users) + add :user_id, references(:users, on_delete: :delete_all), null: false end + + create index(:keywords, [:user_id]) end end diff --git a/test/google_crawler/search/keyword_test.exs b/test/google_crawler/search/keyword_test.exs new file mode 100644 index 0000000..d18e444 --- /dev/null +++ b/test/google_crawler/search/keyword_test.exs @@ -0,0 +1,26 @@ +defmodule Googlecrawler.Search.KeywordTest do + use GoogleCrawler.DataCase + + alias GoogleCrawler.Search.Keyword + alias GoogleCrawler.UserFactory + alias GoogleCrawler.KeywordFactory + + describe "changeset" do + test "keyword is required" do + user = UserFactory.create() + attrs = KeywordFactory.build_attrs(%{keyword: "", user: user}) + changeset = Keyword.changeset(%Keyword{}, attrs) + + refute changeset.valid? + assert %{keyword: ["can't be blank"]} = errors_on(changeset) + end + + test "user is required" do + attrs = KeywordFactory.build_attrs() + changeset = Keyword.changeset(%Keyword{}, attrs) + + refute changeset.valid? + assert %{user_id: ["can't be blank"]} = errors_on(changeset) + end + end +end