Initial elasticsearch changes

Author: paulsullivanjr

config/dev.exs

@@ -51,6 +51,10 @@ config :code_corps, :analytics, CodeCorps.Analytics.InMemoryAPI
 config :code_corps, :stripe, Stripe
 config :code_corps, :stripe_env, :dev
 
+# Configure elasticsearch
+config :code_corps, :elasticsearch_url, "http://0.0.0.0:9200"
+config :code_corps, :elasticsearch_index,  "skills"
+
 config :sentry,
   environment_name: Mix.env || :dev
 

config/prod.exs

@@ -50,6 +50,10 @@ config :code_corps, :analytics, CodeCorps.Analytics.SegmentAPI
 config :code_corps, :stripe, Stripe
 config :code_corps, :stripe_env, :prod
 
+# Configure elasticsearch
+config :code_corps, :elasticsearch_url, "http://0.0.0.0:9200"
+config :code_corps, :elasticsearch_index,  "skills"
+
 config :sentry,
   environment_name: Mix.env || :prod
 

config/test.exs

@@ -36,9 +36,17 @@ config :code_corps, :stripe_env, :test
 
 config :code_corps, :icon_color_generator, CodeCorps.RandomIconColor.TestGenerator
 
+# Configure elasticsearch
+config :code_corps, :elasticsearch_url, "http://0.0.0.0:9200"
+config :code_corps, :elasticsearch_index,  "skills"
+
 # Set Corsica logging to output no console warning when rejecting a request
 config :code_corps, :corsica_log_level, [rejected: :debug]
 
+# Configure elasticsearch
+config :code_corps, :elasticsearch_url, "http://0.0.0.0:9200"
+config :code_corps, :elasticsearch_index,  "skills"
+
 config :sentry,
   environment_name: Mix.env || :test
 

mix.exs

@@ -4,7 +4,7 @@ defmodule CodeCorps.Mixfile do
   def project do
     [app: :code_corps,
      version: "0.0.1",
-     elixir: "1.3.4",
+     elixir: "1.4.0",
      elixirc_paths: elixirc_paths(Mix.env),
      compilers: [:phoenix, :gettext] ++ Mix.compilers,
      build_embedded: Mix.env == :prod,
@@ -94,7 +94,8 @@ defmodule CodeCorps.Mixfile do
       {:timber, "~> 0.4"}, # Logging
       {:timex, "~> 3.0"},
       {:timex_ecto, "~> 3.0"},
-      {:ecto_ordered, "0.2.0-beta1"}
+      {:ecto_ordered, "0.2.0-beta1"},
+      {:elastix, git: "https://github.com/paulsullivanjr/elastix.git"} # for elastic search
     ]
   end
 

mix.lock

@@ -19,6 +19,7 @@
   "earmark": {:hex, :earmark, "1.0.3", "89bdbaf2aca8bbb5c97d8b3b55c5dd0cff517ecc78d417e87f1d0982e514557b", [:mix], []},
   "ecto": {:hex, :ecto, "2.1.1", "fa8bdb14be9992b777036e20f183b8c4300cc012a0fae748529ff89b5423f2dd", [:mix], [{:db_connection, "~> 1.1", [hex: :db_connection, optional: true]}, {:decimal, "~> 1.2", [hex: :decimal, optional: false]}, {:mariaex, "~> 0.8.0", [hex: :mariaex, optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, optional: true]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: false]}, {:postgrex, "~> 0.13.0", [hex: :postgrex, optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, optional: true]}]},
   "ecto_ordered": {:hex, :ecto_ordered, "0.2.0-beta1", "cb066bc608f1c8913cea85af8293261720e6a88e3c99061e6877d7025352f045", [:mix], [{:ecto, "~> 2.0", [hex: :ecto, optional: false]}]},
+  "elastix": {:git, "https://github.com/paulsullivanjr/elastix.git", "72441f08d59491ec1101b8bb9afe56463a5cbd75", []},
   "ex_aws": {:hex, :ex_aws, "1.0.0", "509368ffbd38b405faefda95eb30012fc7b791fad6574ef427cc4ad0f4f5b533", [:mix], [{:configparser_ex, "~> 0.2.1", [hex: :configparser_ex, optional: true]}, {:gen_stage, "~> 0.8.0", [hex: :gen_stage, optional: false]}, {:hackney, "1.6.1", [hex: :hackney, optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, optional: true]}, {:sweet_xml, "~> 0.5", [hex: :sweet_xml, optional: true]}]},
   "ex_doc": {:hex, :ex_doc, "0.14.5", "c0433c8117e948404d93ca69411dd575ec6be39b47802e81ca8d91017a0cf83c", [:mix], [{:earmark, "~> 1.0", [hex: :earmark, optional: false]}]},
   "ex_machina": {:hex, :ex_machina, "1.0.2", "1cc49e1a09e3f7ab2ecb630c17f14c2872dc4ec145d6d05a9c3621936a63e34f", [:mix], [{:ecto, "~> 2.0", [hex: :ecto, optional: true]}]},

test/integration/skill_controller_search_test.exs

@@ -0,0 +1,29 @@
+defmodule SkillControllerSearchIntegrationTest do
+  use ExUnit.Case, async: true
+  alias CodeCorps.ElasticSearchHelper
+
+  @test_url Application.get_env(:code_corps, :elasticsearch_url)
+  @test_index  Application.get_env(:code_corps, :elasticsearch_index)
+
+  setup do
+    Elastix.Index.delete(Application.get_env(:code_corps, :elasticsearch_url),
+      Application.get_env(:code_corps, :elasticsearch_index))
+    :ok
+  end
+
+  test "search partial word" do
+   results = ElasticSearchHelper.search("ru")
+   assert results == ["Ruby"]
+  end
+
+  test "search whole word" do
+    results = ElasticSearchHelper.search("css")
+    assert results == ["CSS"]
+  end
+
+  test "search no matches" do
+    results = ElasticSearchHelper.search("foo")
+    assert results == []
+  end
+
+end

web/helpers/elastic_search_helper.ex

@@ -0,0 +1,108 @@
+defmodule CodeCorps.ElasticSearchHelper do
+  alias Elastix.Search
+  alias Elastix.Index
+  alias Elastix.Document
+
+  @test_url Application.get_env(:code_corps, :elasticsearch_url)
+  @test_index  Application.get_env(:code_corps, :elasticsearch_index)
+
+  def delete(url, index) do
+    Index.delete(url, index)
+  end
+
+  def create_index(url, index, type) do
+    Index.settings(url, index, settings_map)
+    Index.settings(url, "#{index}/_mapping/#{type}", field_filter)
+  end
+
+  def init do
+    Index.settings(@test_url, @test_index, settings_map)
+    Index.settings(@test_url, "skills/_mapping/title", field_filter)
+
+    data = %{
+      title: "CSS"
+    }
+    Document.index @test_url, @test_index, "title", 1, data, [refresh: true]
+    data = %{
+      title: "Elixir"
+    }
+    Document.index @test_url, @test_index, "title", 2, data, [refresh: true]
+    data = %{
+      title: "Ruby"
+    }
+    Document.index @test_url, @test_index, "title", 3, data, [refresh: true]
+
+    data = %{
+      title: "Rails"
+    }
+    Document.index @test_url, @test_index, "title", 4, data, [refresh: true]
+  end
+
+  def search(search_query) do
+    init
+
+    data = %{
+      query: %{
+        match: %{ title: search_query }
+      }
+    }
+
+    response = Search.search @test_url, @test_index, [], data
+
+    #response.status_code == 200 do
+    #count = response.body["hits"]["total"]
+
+    hits = response.body["hits"]["hits"] || []
+    Enum.map(hits, fn(x) -> x["_source"]["title"] end)
+  end
+
+  defp settings_map do
+  %{
+      settings: %{
+        number_of_shards: 5,
+        analysis: %{
+          filter: %{
+            autocomplete_filter: %{
+              type:     "edge_ngram",
+              min_gram: 2,
+              max_gram: 20
+            }
+          },
+          analyzer: %{
+            autocomplete: %{
+              type:      "custom",
+              tokenizer: "standard",
+              filter: [
+                "lowercase",
+                "autocomplete_filter"
+              ]
+            }
+          }
+        }
+      }
+    }
+  end
+
+  defp field_filter do
+    %{
+      title: %{
+        properties: %{
+          title: %{
+            "type":     "string",
+            "analyzer": "autocomplete"
+          }
+        }
+      }
+    }
+  end
+end
+
+# elastix index.ex gets:
+  # custom method
+  #  @doc false
+  #  def settings(elastic_url, name, data) do
+    #  elastic_url <> make_path(name)
+    # |> HTTP.put(Poison.encode!(data))
+    #    |> process_response
+    #  end
+