Add bulk create backend for local authorities

Author: mbklein

app/lib/meadow_web/controllers/authority_records_controller.ex

@@ -7,6 +7,57 @@ defmodule MeadowWeb.AuthorityRecordsController do
 
   plug(:authorize_user)
 
+  def bulk_create(conn, %{"records" => %Plug.Upload{path: path}}) do
+    csv_to_maps(path)
+    |> do_bulk_create(conn)
+  end
+
+  defp do_bulk_create({:error, :bad_format}, conn) do
+    send_resp(conn, 400, "Bad Request")
+  end
+
+  defp do_bulk_create({:ok, records}, conn) do
+    results =
+      records
+      |> AuthorityRecords.create_authority_records()
+      |> Enum.map(fn {status, %{id: id, label: label, hint: hint}} ->
+        ["info:nul/#{id}", label, hint, status]
+      end)
+
+    file = "authority_import_#{DateTime.utc_now() |> DateTime.to_unix()}.csv"
+
+    conn =
+      conn
+      |> put_resp_content_type("text/csv")
+      |> put_resp_header("content-disposition", ~s[attachment; filename="#{file}"])
+      |> send_chunked(:ok)
+
+    [~w(id label hint status) | results]
+    |> CSV.dump_to_stream()
+    |> Stream.each(fn csv_row ->
+      chunk(conn, csv_row)
+    end)
+    |> Stream.run()
+
+    conn
+  end
+
+  defp csv_to_maps(file) do
+    [headers | rows] =
+      File.stream!(file, [], :line)
+      |> CSV.parse_stream(skip_headers: false)
+      |> Enum.to_list()
+
+    case Enum.sort(headers) do
+      ~w(hint label) ->
+        headers = Enum.map(headers, &String.to_atom/1)
+        {:ok, Enum.map(rows, fn row -> Enum.zip(headers, row) |> Enum.into(%{}) end)}
+
+      _ ->
+        {:error, :bad_format}
+    end
+  end
+
   def export(conn, %{"file" => file} = params) do
     export(conn, Path.extname(file), params)
   end

app/lib/meadow_web/router.ex

@@ -56,6 +56,7 @@ defmodule MeadowWeb.Router do
     pipe_through(:api)
 
     post("/export/:file", MeadowWeb.ExportController, :export)
+    post("/authority_records/bulk_create", MeadowWeb.AuthorityRecordsController, :bulk_create)
     post("/authority_records/:file", MeadowWeb.AuthorityRecordsController, :export)
     post("/create_shared_links/:file", MeadowWeb.SharedLinksController, :export)
 

app/lib/nul/authority_records.ex

@@ -3,6 +3,7 @@ defmodule NUL.AuthorityRecords do
   The NUL.AuthorityRecords context.
   """
 
+  #  alias Faker.NaiveDateTime
   alias Meadow.Repo
   alias NUL.Schemas.AuthorityRecord
 
@@ -90,6 +91,47 @@ defmodule NUL.AuthorityRecords do
     |> Repo.insert!()
   end
 
+  @doc """
+  Creates many AuthorityRecords at once. Returns a list of [{:created|:duplicate}, %AuthorityRecord{}]
+  where the record is either the newly created record or the retrieved existing record
+  """
+  def create_authority_records(list_of_attrs) do
+    inserted_at = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)
+
+    records =
+      Enum.map(list_of_attrs, fn attrs ->
+        Map.merge(attrs, %{
+          id: Ecto.UUID.generate(),
+          inserted_at: inserted_at,
+          updated_at: inserted_at
+        })
+      end)
+
+    labels = Enum.map(records, & &1.label)
+
+    duplicates =
+      from(ar in AuthorityRecord, where: ar.label in ^labels)
+      |> Repo.all()
+      |> indexed_records(:duplicate)
+
+    created =
+      Repo.insert_all(AuthorityRecord, records,
+        returning: true,
+        on_conflict: :nothing,
+        conflict_target: :label
+      )
+      |> indexed_records(:created)
+
+    results = Enum.into(created ++ duplicates, %{})
+    Enum.map(labels, &Map.get(results, &1))
+  end
+
+  # def create_authority_records(list_of_attrs) do
+  #   Repo.transaction(fn ->
+  #     {:ok, Enum.map(list_of_attrs, &create_authority_record/1)} |> IO.inspect()
+  #   end)
+  # end
+
   @doc """
   Updates an AuthorityRecord.
 
@@ -114,4 +156,13 @@ defmodule NUL.AuthorityRecords do
   def delete_authority_record(%AuthorityRecord{} = authority_record) do
     Repo.delete(authority_record)
   end
+
+  defp indexed_records({_, records}, status), do: indexed_records(records, status)
+
+  defp indexed_records(records, status) do
+    records
+    |> Enum.map(fn record ->
+      {record.label, {status, record}}
+    end)
+  end
 end

app/test/fixtures/authority_records/bad_authority_import.csv

@@ -0,0 +1,4 @@
+not label,hint
+"First Imported Thing",created by test
+"Second Imported Thing",created by test
+"Third Imported Thing",created by test

app/test/fixtures/authority_records/good_authority_import.csv

@@ -0,0 +1,4 @@
+label,hint
+"First Imported Thing",created by test
+"Second Imported Thing",created by test
+"Third Imported Thing",created by test

app/test/meadow_web/controllers/authority_records_controller_test.exs

@@ -2,6 +2,8 @@ defmodule MeadowWeb.AuthorityRecordsControllerTest do
   use Meadow.DataCase
   use MeadowWeb.ConnCase, async: true
 
+  alias Meadow.Repo
+  alias NUL.AuthorityRecords
   alias NUL.Schemas.AuthorityRecord
 
   describe "POST /api/authority_records/:filename (failure)" do
@@ -45,6 +47,7 @@ defmodule MeadowWeb.AuthorityRecordsControllerTest do
         authority_record_fixture(%{label: "Ver Steeg, Dorothy A."}),
         authority_record_fixture(%{label: "Netsch, Walter A."})
       ]
+
       :ok
     end
 
@@ -62,4 +65,61 @@ defmodule MeadowWeb.AuthorityRecordsControllerTest do
       assert response(conn, 200)
     end
   end
+
+  describe "POST /api/authority_records/bulk_create" do
+    setup %{fixture: fixture} do
+      upload = %Plug.Upload{
+        content_type: "text/csv",
+        filename: "authority_import.csv",
+        path: fixture
+      }
+
+      {:ok, %{upload: upload}}
+    end
+
+    @tag fixture: "test/fixtures/authority_records/bad_authority_import.csv"
+    test "bad data", %{conn: conn, upload: upload} do
+      conn =
+        conn
+        |> auth_user(user_fixture("TestAdmins"))
+        |> post("/api/authority_records/bulk_create", %{records: upload})
+
+      assert response(conn, 400)
+    end
+
+    @tag fixture: "test/fixtures/authority_records/good_authority_import.csv"
+    test "good data", %{conn: conn, upload: upload} do
+      precount = Meadow.Repo.aggregate(AuthorityRecord, :count)
+
+      conn =
+        conn
+        |> auth_user(user_fixture("TestAdmins"))
+        |> post("/api/authority_records/bulk_create", %{records: upload})
+
+      assert Meadow.Repo.aggregate(AuthorityRecord, :count) == precount + 3
+      assert conn.state == :chunked
+      assert response_content_type(conn, :csv)
+      assert response(conn, 200)
+    end
+
+    @tag fixture: "test/fixtures/authority_records/good_authority_import.csv"
+    test "one duplicate entry", %{conn: conn, upload: upload} do
+      AuthorityRecords.create_authority_record(%{
+        label: "Second Imported Thing",
+        hint: "preexisting entry"
+      })
+
+      precount = Meadow.Repo.aggregate(AuthorityRecord, :count)
+
+      conn =
+        conn
+        |> auth_user(user_fixture("TestAdmins"))
+        |> post("/api/authority_records/bulk_create", %{records: upload})
+
+      assert Meadow.Repo.aggregate(AuthorityRecord, :count) == precount + 2
+      assert conn.state == :chunked
+      assert response_content_type(conn, :csv)
+      assert response(conn, 200)
+    end
+  end
 end