Add /robots.txt route

lib/httparrot.ex

@@ -24,6 +24,7 @@ defmodule HTTParrot do
              {'/delay/:n', HTTParrot.DelayedHandler, []},
              {'/html', :cowboy_static, {:priv_file, :httparrot, "html.html"}},
              {'/deny', HTTParrot.DenyHandler, []},
+             {'/robots.txt', HTTParrot.RobotsHandler, []},
              {'/base64/:value', HTTParrot.Base64Handler, []} ] }
     ])
     {:ok, http_port} = :application.get_env(:httparrot, :http_port)

lib/httparrot/robots_handler.ex

@@ -0,0 +1,26 @@
+defmodule HTTParrot.RobotsHandler do
+  @moduledoc """
+  Returns a robots.txt.
+  """
+
+  def init(_transport, _req, _opts) do
+    {:upgrade, :protocol, :cowboy_rest}
+  end
+
+  def allowed_methods(req, state) do
+    {["GET", "HEAD", "OPTIONS"], req, state}
+  end
+
+  def content_types_provided(req, state) do
+    {[{{"text", "plain", []}, :get_text}], req, state}
+  end
+
+  @robots """
+  User-agent: *
+  Disallow: /deny
+  """
+
+  def get_text(req, state), do: {@robots, req, state}
+
+  def terminate(_, _, _), do: :ok
+end