diff --git a/.formatter.exs b/.formatter.exs index 4763d0a0..658fc4d0 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,7 +1,7 @@ [ inputs: [ "{mix,.formatter}.exs", - "{config,lib,test}/**/*.{ex,exs}" + "{config,bench,lib,test}/**/*.{ex,exs}" ], line_length: 88 ] diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e7168e0..3c4a10a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Unreleased + +- removed: `db_connection`. It's just a NIF now. `db_connection` is moved to `ecto_sqlite3` +- removed: `Exqlite.Basic` + ## v0.24.1 - fixed: Pre-compile images for Apple and Windows diff --git a/README.md b/README.md index a438869b..71866d65 100644 --- a/README.md +++ b/README.md @@ -15,42 +15,24 @@ Package: https://hex.pm/packages/exqlite ## Caveats -* Prepared statements are not cached. * Prepared statements are not immutable. You must be careful when manipulating statements and binding values to statements. Do not try to manipulate the statements concurrently. Keep it isolated to one process. -* Simultaneous writing is not supported by SQLite3 and will not be supported - here. -* All native calls are run through the Dirty NIF scheduler. -* Datetimes are stored without offsets. This is due to how SQLite3 handles date - and times. If you would like to store a timezone, you will need to create a - second column somewhere storing the timezone name and shifting it when you - get it from the database. This is more reliable than storing the offset as - `+03:00` as it does not respect daylight savings time. -* When storing `BLOB` values, you have to use `{:blob, the_binary}`, otherwise - it will be interpreted as a string. +* Some native calls are run through the Dirty NIF scheduler. + Some are executed directly on current scheduler. ## Installation ```elixir defp deps do [ - {:exqlite, "~> 0.23"} + {:exqlite, "~> 1.0"} ] end ``` ## Configuration - -### Runtime Configuration - -```elixir -config :exqlite, default_chunk_size: 100 -``` - -* `default_chunk_size` - The chunk size that is used when multi-stepping when - not specifying the chunk size explicitly. ### Compile-time Configuration @@ -126,47 +108,52 @@ export EXQLITE_SYSTEM_CFLAGS=-I/usr/local/include/sqlcipher export EXQLITE_SYSTEM_LDFLAGS=-L/usr/local/lib -lsqlcipher ``` -Once you have `exqlite` configured, you can use the `:key` option in the database config to enable encryption: +Once you have `exqlite` build configured, you can use the `key` pragma to enable encryption: ```elixir -config :exqlite, key: "super-secret' +{:ok, db} = Exqlite.open("sqlcipher.db") +:ok = Exqlite.execute(db, "pragma key='super-secret'") ``` ## Usage -The `Exqlite.Sqlite3` module usage is fairly straight forward. +The `Exqlite` module usage is fairly straight forward. ```elixir -# We'll just keep it in memory right now -{:ok, conn} = Exqlite.Sqlite3.open(":memory:") +{:ok, db} = Exqlite.open("app.db", [:readwrite, :create]) + +:ok = Exqlite.execute(db, "pragma foreign_keys=on") +:ok = Exqlite.execute(db, "pragma journal_mode=wal") +:ok = Exqlite.execute(db, "pragma busy_timeout=5000") # Create the table -:ok = Exqlite.Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") +:ok = Exqlite.execute(db, "create table test (id integer primary key, stuff text)") # Prepare a statement -{:ok, statement} = Exqlite.Sqlite3.prepare(conn, "insert into test (stuff) values (?1)") -:ok = Exqlite.Sqlite3.bind(conn, statement, ["Hello world"]) +{:ok, insert} = Exqlite.prepare(db, "insert into test (stuff) values (?1)") +:ok = Exqlite.bind_all(db, insert, ["Hello world"]) # Step is used to run statements -:done = Exqlite.Sqlite3.step(conn, statement) +:done = Exqlite.step(db, insert) # Prepare a select statement -{:ok, statement} = Exqlite.Sqlite3.prepare(conn, "select id, stuff from test") +{:ok, select} = Exqlite.prepare(db, "select id, stuff from test") # Get the results -{:row, [1, "Hello world"]} = Exqlite.Sqlite3.step(conn, statement) +{:row, [1, "Hello world"]} = Exqlite.step(db, select) # No more results -:done = Exqlite.Sqlite3.step(conn, statement) +:done = Exqlite.step(db, select) -# Release the statement. +# Release the statements. # # It is recommended you release the statement after using it to reclaim the memory # asap, instead of letting the garbage collector eventually releasing the statement. # # If you are operating at a high load issuing thousands of statements, it would be # possible to run out of memory or cause a lot of pressure on memory. -:ok = Exqlite.Sqlite3.release(conn, statement) +:ok = Exqlite.finalize(insert) +:ok = Exqlite.finalize(select) ``` ### Using SQLite3 native extensions @@ -177,52 +164,39 @@ available by installing the [ExSqlean](https://github.com/mindreframer/ex_sqlean package. This package wraps [SQLean: all the missing SQLite functions](https://github.com/nalgeon/sqlean). ```elixir -alias Exqlite.Basic -{:ok, conn} = Basic.open("db.sqlite3") -:ok = Basic.enable_load_extension(conn) +{:ok, db} = Exqlite.open(":memory:", [:readwrite]) +:ok = Exqlite.enable_load_extension(db, true) + +exec = fn db, sql, params -> + with {:ok, stmt} <- Exqlite.prepare(db, sql) do + try do + with :ok <- Exqlite.bind_all(db, stmt, params) do + Exqlite.fetch_all(db, stmt) + end + after + Exqlite.finalize(stmt) + end + end +end # load the regexp extension - https://github.com/nalgeon/sqlean/blob/main/docs/re.md -Basic.load_extension(conn, ExSqlean.path_for("re")) +{:ok, _rows} = exec.(db, "select load_extension(?)", [ExSqlean.path_for("re")]) # run some queries to test the new `regexp_like` function -{:ok, [[1]], ["value"]} = Basic.exec(conn, "select regexp_like('the year is 2021', ?) as value", ["2021"]) |> Basic.rows() -{:ok, [[0]], ["value"]} = Basic.exec(conn, "select regexp_like('the year is 2021', ?) as value", ["2020"]) |> Basic.rows() +{:ok, [[1]], ["value"]} = exec.(db, "select regexp_like('the year is 2021', ?) as value", ["2021"]) +{:ok, [[0]], ["value"]} = exec.(db, "select regexp_like('the year is 2021', ?) as value", ["2020"]) # prevent loading further extensions -:ok = Basic.disable_load_extension(conn) -{:error, %Exqlite.Error{message: "not authorized"}, _} = Basic.load_extension(conn, ExSqlean.path_for("re")) +:ok = Exqlite.enable_load_extension(db, false) -# close connection -Basic.close(conn) -``` - -It is also possible to load extensions using the `Connection` configuration. For example: +{:error, %Exqlite.Error{message: "not authorized"}} = + exec.(db, "select load_extension(?)", [ExSqlean.path_for("stats")]) -```elixir -arch_dir = - System.cmd("uname", ["-sm"]) - |> elem(0) - |> String.trim() - |> String.replace(" ", "-") - |> String.downcase() # => "darwin-arm64" - -config :myapp, arch_dir: arch_dir - -# global -config :exqlite, load_extensions: [ "./priv/sqlite/\#{arch_dir}/rotate" ] - -# per connection in a Phoenix app -config :myapp, Myapp.Repo, - database: "path/to/db", - load_extensions: [ - "./priv/sqlite/\#{arch_dir}/vector0", - "./priv/sqlite/\#{arch_dir}/vss0" - ] +# close connection +Exqlite.close(db) ``` -See [Exqlite.Connection.connect/1](https://hexdocs.pm/exqlite/Exqlite.Connection.html#connect/1) -for more information. When using extensions for SQLite3, they must be compiled -for the environment you are targeting. +When using extensions for SQLite3, they must be compiled for the environment you are targeting. ## Why SQLite3 @@ -239,7 +213,7 @@ that would be resiliant to power outages and still maintain some state that ## Under The Hood -We are using the Dirty NIF scheduler to execute the sqlite calls. The rationale +We are using the Dirty NIF scheduler to execute most of the sqlite calls. The rationale behind this is that maintaining each sqlite's connection command pool is complicated and error prone. diff --git a/bench/bind.exs b/bench/bind.exs new file mode 100644 index 00000000..3fd57974 --- /dev/null +++ b/bench/bind.exs @@ -0,0 +1,7 @@ +{:ok, db} = Exqlite.open(":memory:", [:readwrite, :nomutex]) +{:ok, stmt} = Exqlite.prepare(db, "select ? + 1") + +Benchee.run(%{ + "bind_all" => fn -> Exqlite.bind_all(db, stmt, [1]) end, + "dirty_cpu_bind_all" => fn -> Exqlite.dirty_cpu_bind_all(db, stmt, [1]) end +}) diff --git a/bench/insert_all.exs b/bench/insert_all.exs new file mode 100644 index 00000000..66360cf6 --- /dev/null +++ b/bench/insert_all.exs @@ -0,0 +1,18 @@ +{:ok, db} = Exqlite.open(":memory:", [:readwrite]) +:ok = Exqlite.execute(db, "create table test (id integer primary key, name text)") +{:ok, stmt} = Exqlite.prepare(db, "insert into test(name) values(?)") + +Benchee.run( + %{ + "insert_all" => + {fn rows -> Exqlite.insert_all(db, stmt, rows) end, + before_scenario: fn _input -> Exqlite.execute(db, "truncate test") end} + }, + inputs: %{ + "3 rows" => Enum.map(1..3, fn i -> ["name-#{i}"] end), + "30 rows" => Enum.map(1..30, fn i -> ["name-#{i}"] end), + "90 rows" => Enum.map(1..90, fn i -> ["name-#{i}"] end), + "300 rows" => Enum.map(1..300, fn i -> ["name-#{i}"] end), + "1000 rows" => Enum.map(1..1000, fn i -> ["name-#{i}"] end) + } +) diff --git a/bench/prepare.exs b/bench/prepare.exs new file mode 100644 index 00000000..e69de29b diff --git a/bench/step.exs b/bench/step.exs new file mode 100644 index 00000000..884afc5e --- /dev/null +++ b/bench/step.exs @@ -0,0 +1,58 @@ +tmp_dir = Path.expand(Path.join("./tmp", "bench/step")) +File.mkdir_p!(tmp_dir) + +path = Path.join(tmp_dir, "db.sqlite") +if File.exists?(path), do: File.rm!(path) + +IO.puts("Creating DB at #{path} ...") +{:ok, db} = Exqlite.open(path, [:readwrite, :nomutex, :create]) + +IO.puts("Inserting 1000 rows ...") +:ok = Exqlite.execute(db, "create table test(stuff text)") +{:ok, insert} = Exqlite.prepare(db, "insert into test(stuff) values(?)") +:ok = Exqlite.insert_all(db, insert, Enum.map(1..1000, fn i -> ["name-#{i}"] end)) +:ok = Exqlite.finalize(insert) + +select = fn limit -> + {:ok, select} = Exqlite.prepare(db, "select * from test limit #{limit}") + select +end + +defmodule Bench do + def step_all(db, stmt) do + case Exqlite.step(db, stmt) do + {:row, _} -> step_all(db, stmt) + :done -> :ok + end + end + + def dirty_io_step_all(db, stmt) do + case Exqlite.dirty_io_step(db, stmt) do + {:row, _} -> dirty_io_step_all(db, stmt) + :done -> :ok + end + end + + def multi_step_all(db, stmt, steps) do + case Exqlite.multi_step(db, stmt, steps) do + {:rows, _} -> multi_step_all(db, stmt, steps) + {:done, _} -> :ok + end + end +end + +IO.puts("Running benchmarks ...\n") + +Benchee.run( + %{ + "step" => fn stmt -> Bench.step_all(db, stmt) end, + "dirty_io_step" => fn stmt -> Bench.dirty_io_step_all(db, stmt) end, + "multi_step(100)" => fn stmt -> Bench.multi_step_all(db, stmt, _steps = 100) end + }, + inputs: %{ + "10 rows" => select.(10), + "100 rows" => select.(100), + "500 rows" => select.(500) + }, + memory_time: 2 +) diff --git a/c_src/sqlite3_nif.c b/c_src/sqlite3_nif.c index 4418d6b8..fa416399 100644 --- a/c_src/sqlite3_nif.c +++ b/c_src/sqlite3_nif.c @@ -28,7 +28,6 @@ typedef struct connection typedef struct statement { - connection_t* conn; sqlite3_stmt* statement; } statement_t; @@ -109,20 +108,6 @@ exqlite_mem_shutdown(void* ptr) { } -static const char* -get_sqlite3_error_msg(int rc, sqlite3* db) -{ - if (rc == SQLITE_MISUSE) { - return "Sqlite3 was invoked incorrectly."; - } - - const char* message = sqlite3_errmsg(db); - if (!message) { - return "No error message available."; - } - return message; -} - static ERL_NIF_TERM make_atom(ErlNifEnv* env, const char* atom_name) { @@ -148,12 +133,12 @@ make_ok_tuple(ErlNifEnv* env, ERL_NIF_TERM value) } static ERL_NIF_TERM -make_error_tuple(ErlNifEnv* env, const char* reason) +raise_exception(ErlNifEnv* env, const char* reason) { assert(env); assert(reason); - return enif_make_tuple2(env, make_atom(env, "error"), make_atom(env, reason)); + return enif_raise_exception(env, enif_make_string(env, reason, ERL_NIF_LATIN1)); } static ERL_NIF_TERM @@ -176,13 +161,16 @@ make_binary(ErlNifEnv* env, const void* bytes, unsigned int size) static ERL_NIF_TERM make_sqlite3_error_tuple(ErlNifEnv* env, int rc, sqlite3* db) { - const char* msg = get_sqlite3_error_msg(rc, db); - size_t len = strlen(msg); + const char* msg = sqlite3_errmsg(db); + + if (!msg) + msg = sqlite3_errstr(rc); - return enif_make_tuple2( + return enif_make_tuple3( env, make_atom(env, "error"), - make_binary(env, msg, len)); + enif_make_int64(env, rc), + make_binary(env, msg, strlen(msg))); } static ERL_NIF_TERM @@ -197,6 +185,7 @@ exqlite_open(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) sqlite3* db = NULL; ErlNifMutex* mutex = NULL; ERL_NIF_TERM result; + // TODO ErlNifBinary bin; ERL_NIF_TERM eos = enif_make_int(env, 0); @@ -206,22 +195,27 @@ exqlite_open(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } if (!enif_inspect_iolist_as_binary(env, enif_make_list2(env, argv[0], eos), &bin)) { - return make_error_tuple(env, "invalid_filename"); + return raise_exception(env, "invalid filename"); } if (!enif_get_int(env, argv[1], &flags)) { - return make_error_tuple(env, "invalid_flags"); + return raise_exception(env, "invalid flags"); } rc = sqlite3_open_v2((char*)bin.data, &db, flags, NULL); if (rc != SQLITE_OK) { - return make_error_tuple(env, "database_open_failed"); + const char* msg = sqlite3_errstr(rc); + return enif_make_tuple3( + env, + make_atom(env, "error"), + enif_make_int64(env, rc), + make_binary(env, msg, strlen(msg))); } mutex = enif_mutex_create("exqlite:connection"); if (mutex == NULL) { sqlite3_close_v2(db); - return make_error_tuple(env, "failed_to_create_mutex"); + return raise_exception(env, "failed to create mutex"); } sqlite3_busy_timeout(db, 2000); @@ -230,7 +224,7 @@ exqlite_open(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) if (!conn) { sqlite3_close_v2(db); enif_mutex_destroy(mutex); - return make_error_tuple(env, "out_of_memory"); + return raise_exception(env, "falied to allocate connection resource"); } conn->db = db; conn->mutex = mutex; @@ -254,7 +248,7 @@ exqlite_close(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) { - return make_error_tuple(env, "invalid_connection"); + return raise_exception(env, "invalid connection"); } // DB is already closed, nothing to do here @@ -310,11 +304,11 @@ exqlite_execute(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) { - return make_error_tuple(env, "invalid_connection"); + return raise_exception(env, "invalid connection"); } if (!enif_inspect_iolist_as_binary(env, enif_make_list2(env, argv[1], eos), &bin)) { - return make_error_tuple(env, "sql_not_iolist"); + return raise_exception(env, "sql not iodata"); } rc = sqlite3_exec(conn->db, (char*)bin.data, NULL, NULL, NULL); @@ -340,11 +334,11 @@ exqlite_changes(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) { - return make_error_tuple(env, "invalid_connection"); + return raise_exception(env, "invalid connection"); } if (conn->db == NULL) { - return make_error_tuple(env, "connection_closed"); + return raise_exception(env, "connection closed"); } int changes = sqlite3_changes(conn->db); @@ -371,28 +365,25 @@ exqlite_prepare(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) { - return make_error_tuple(env, "invalid_connection"); + return raise_exception(env, "invalid connection"); } if (!enif_inspect_iolist_as_binary(env, enif_make_list2(env, argv[1], eos), &bin)) { - return make_error_tuple(env, "sql_not_iolist"); + return raise_exception(env, "sql not iodata"); } statement = enif_alloc_resource(statement_type, sizeof(statement_t)); if (!statement) { - return make_error_tuple(env, "out_of_memory"); + return raise_exception(env, "failed to allocate statement resource"); } statement->statement = NULL; - enif_keep_resource(conn); - statement->conn = conn; - // ensure connection is not getting closed by parallel thread enif_mutex_lock(conn->mutex); if (conn->db == NULL) { enif_mutex_unlock(conn->mutex); enif_release_resource(statement); - return make_error_tuple(env, "connection_closed"); + return raise_exception(env, "connection closed"); } rc = sqlite3_prepare_v3(conn->db, (char*)bin.data, bin.size, 0, &statement->statement, NULL); enif_mutex_unlock(conn->mutex); @@ -464,7 +455,7 @@ bind(ErlNifEnv* env, const ERL_NIF_TERM arg, sqlite3_stmt* statement, int index) /// @brief Binds arguments to the sql statement /// static ERL_NIF_TERM -exqlite_bind(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +exqlite_bind_all(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { assert(env); @@ -481,20 +472,20 @@ exqlite_bind(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) { - return make_error_tuple(env, "invalid_connection"); + return raise_exception(env, "invalid connection"); } if (!enif_get_resource(env, argv[1], statement_type, (void**)&statement)) { - return make_error_tuple(env, "invalid_statement"); + return raise_exception(env, "invalid statement"); } if (!enif_get_list_length(env, argv[2], &argument_list_length)) { - return make_error_tuple(env, "bad_argument_list"); + return raise_exception(env, "bad argument list"); } parameter_count = (unsigned int)sqlite3_bind_parameter_count(statement->statement); if (parameter_count != argument_list_length) { - return make_error_tuple(env, "arguments_wrong_length"); + return raise_exception(env, "arguments wrong length"); } sqlite3_reset(statement->statement); @@ -504,6 +495,7 @@ exqlite_bind(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) enif_get_list_cell(env, list, &head, &tail); int rc = bind(env, head, statement->statement, i + 1); if (rc == -1) { + // TODO return enif_make_tuple2( env, make_atom(env, "error"), @@ -549,6 +541,7 @@ make_cell(ErlNifEnv* env, sqlite3_stmt* statement, unsigned int i) sqlite3_column_bytes(statement, i)); default: + // TODO return make_atom(env, "unsupported"); } } @@ -565,7 +558,8 @@ make_row(ErlNifEnv* env, sqlite3_stmt* statement) columns = enif_alloc(sizeof(ERL_NIF_TERM) * count); if (!columns) { - return make_error_tuple(env, "out_of_memory"); + // TODO + return raise_exception(env, "out of memory"); } for (unsigned int i = 0; i < count; i++) { @@ -593,23 +587,23 @@ exqlite_multi_step(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) { - return make_error_tuple(env, "invalid_connection"); + return raise_exception(env, "invalid connection"); } if (!enif_get_resource(env, argv[1], statement_type, (void**)&statement)) { - return make_error_tuple(env, "invalid_statement"); + return raise_exception(env, "invalid statement"); } if (!statement || !statement->statement) { - return make_error_tuple(env, "invalid_statement"); + return raise_exception(env, "invalid statement"); } if (!enif_get_int(env, argv[2], &chunk_size)) { - return make_error_tuple(env, "invalid_chunk_size"); + return raise_exception(env, "invalid chunk size"); } if (chunk_size < 1) { - return make_error_tuple(env, "invalid_chunk_size"); + return raise_exception(env, "invalid chunk size"); } ERL_NIF_TERM rows = enif_make_list_from_array(env, NULL, 0); @@ -618,10 +612,6 @@ exqlite_multi_step(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) int rc = sqlite3_step(statement->statement); switch (rc) { - case SQLITE_BUSY: - sqlite3_reset(statement->statement); - return make_atom(env, "busy"); - case SQLITE_DONE: return enif_make_tuple2(env, make_atom(env, "done"), rows); @@ -639,6 +629,92 @@ exqlite_multi_step(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) return enif_make_tuple2(env, make_atom(env, "rows"), rows); } +// this function performs a bulk insert of rows by reusing a single prepared statement: +// +// BEGIN IMMEDIATE; +// INSERT INTO table (column1, column2, column3) VALUES (1, 2, 3); +// INSERT INTO table (column1, column2, column3) VALUES (4, 5, 6); +// INSERT INTO table (column1, column2, column3) VALUES (7, 8, 9); +// COMMIT; +static ERL_NIF_TERM +exqlite_insert_all(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + assert(env); + + statement_t* statement = NULL; + connection_t* conn = NULL; + int rc; + + if (argc != 3) { + return enif_make_badarg(env); + } + + if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) { + return raise_exception(env, "invalid connection"); + } + + if (!enif_get_resource(env, argv[1], statement_type, (void**)&statement)) { + return raise_exception(env, "invalid statement"); + } + + if (!enif_is_list(env, argv[2])) { + return raise_exception(env, "expected a list of rows"); + } + + int param_count = (unsigned int)sqlite3_bind_parameter_count(statement->statement); + + ERL_NIF_TERM rows = argv[2]; + ERL_NIF_TERM head, tail; + + if (enif_is_empty_list(env, rows)) { + return make_atom(env, "ok"); // No rows to insert, return early + } + + // Start transaction + rc = sqlite3_exec(conn->db, "BEGIN IMMEDIATE", NULL, NULL, NULL); + if (rc != SQLITE_OK) { + return make_sqlite3_error_tuple(env, rc, conn->db); + } + + while (enif_get_list_cell(env, rows, &head, &tail)) { + sqlite3_reset(statement->statement); + + // Bind row + for (unsigned int i = 1; i <= param_count; i++) { + ERL_NIF_TERM param; + + if (!enif_get_list_cell(env, head, ¶m, &head)) { + sqlite3_exec(conn->db, "ROLLBACK", NULL, NULL, NULL); + return raise_exception(env, "invalid row"); + } + + rc = bind(env, param, statement->statement, i); + + if (rc != SQLITE_OK) { + sqlite3_exec(conn->db, "ROLLBACK", NULL, NULL, NULL); + return make_sqlite3_error_tuple(env, rc, conn->db); + } + } + + // Execute statement + rc = sqlite3_step(statement->statement); + if (rc != SQLITE_DONE) { + sqlite3_exec(conn->db, "ROLLBACK", NULL, NULL, NULL); + return make_sqlite3_error_tuple(env, rc, conn->db); + } + + rows = tail; // Move to the next row + } + + // Commit transaction + rc = sqlite3_exec(conn->db, "COMMIT", NULL, NULL, NULL); + if (rc != SQLITE_OK) { + return make_sqlite3_error_tuple(env, rc, conn->db); + } + + return make_atom(env, "ok"); +} + static ERL_NIF_TERM exqlite_step(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { @@ -652,11 +728,11 @@ exqlite_step(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) { - return make_error_tuple(env, "invalid_connection"); + return raise_exception(env, "invalid connection"); } if (!enif_get_resource(env, argv[1], statement_type, (void**)&statement)) { - return make_error_tuple(env, "invalid_statement"); + return raise_exception(env, "invalid statement"); } int rc = sqlite3_step(statement->statement); @@ -666,8 +742,6 @@ exqlite_step(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) env, make_atom(env, "row"), make_row(env, statement->statement)); - case SQLITE_BUSY: - return make_atom(env, "busy"); case SQLITE_DONE: return make_atom(env, "done"); default: @@ -691,30 +765,30 @@ exqlite_columns(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) { - return make_error_tuple(env, "invalid_connection"); + return raise_exception(env, "invalid connection"); } if (!enif_get_resource(env, argv[1], statement_type, (void**)&statement)) { - return make_error_tuple(env, "invalid_statement"); + return raise_exception(env, "invalid statement"); } size = sqlite3_column_count(statement->statement); if (size == 0) { return make_ok_tuple(env, enif_make_list(env, 0)); } else if (size < 0) { - return make_error_tuple(env, "invalid_column_count"); + return raise_exception(env, "invalid column count"); } columns = enif_alloc(sizeof(ERL_NIF_TERM) * size); if (!columns) { - return make_error_tuple(env, "out_of_memory"); + return raise_exception(env, "out of memory"); } for (int i = 0; i < size; i++) { const char* name = sqlite3_column_name(statement->statement, i); if (!name) { enif_free(columns); - return make_error_tuple(env, "out_of_memory"); + return raise_exception(env, "out of memory"); } columns[i] = make_binary(env, name, strlen(name)); @@ -738,7 +812,7 @@ exqlite_last_insert_rowid(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) { - return make_error_tuple(env, "invalid_connection"); + return raise_exception(env, "invalid connection"); } sqlite3_int64 last_rowid = sqlite3_last_insert_rowid(conn->db); @@ -757,20 +831,20 @@ exqlite_transaction_status(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) { - return make_error_tuple(env, "invalid_connection"); + return raise_exception(env, "invalid connection"); } + // TODO // If the connection times out, DbConnection disconnects the client // and then re-opens a new connection. There is a condition where by // the connection's database is not set but the calling elixir / erlang // pass an incomplete reference. if (!conn->db) { - return make_ok_tuple(env, make_atom(env, "error")); + return raise_exception(env, "invalid connection"); } + int autocommit = sqlite3_get_autocommit(conn->db); - return make_ok_tuple( - env, - autocommit == 0 ? make_atom(env, "transaction") : make_atom(env, "idle")); + return autocommit == 0 ? make_atom(env, "transaction") : make_atom(env, "idle"); } static ERL_NIF_TERM @@ -790,16 +864,16 @@ exqlite_serialize(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) { - return make_error_tuple(env, "invalid_connection"); + return raise_exception(env, "invalid connection"); } if (!enif_inspect_iolist_as_binary(env, enif_make_list2(env, argv[1], eos), &database_name)) { - return make_error_tuple(env, "database_name_not_iolist"); + return raise_exception(env, "schema name not iodata"); } buffer = sqlite3_serialize(conn->db, (char*)database_name.data, &buffer_size, 0); if (!buffer) { - return make_error_tuple(env, "serialization_failed"); + return raise_exception(env, "serialization failed"); } serialized = make_binary(env, buffer, buffer_size); @@ -827,11 +901,11 @@ exqlite_deserialize(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) { - return make_error_tuple(env, "invalid_connection"); + return raise_exception(env, "invalid connection"); } if (!enif_inspect_iolist_as_binary(env, enif_make_list2(env, argv[1], eos), &database_name)) { - return make_error_tuple(env, "database_name_not_iolist"); + return raise_exception(env, "schema name not iodata"); } if (!enif_inspect_binary(env, argv[2], &serialized)) { @@ -841,7 +915,7 @@ exqlite_deserialize(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) size = serialized.size; buffer = sqlite3_malloc(size); if (!buffer) { - return make_error_tuple(env, "deserialization_failed"); + return raise_exception(env, "failed to allocate memory"); } memcpy(buffer, serialized.data, size); @@ -854,23 +928,18 @@ exqlite_deserialize(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } static ERL_NIF_TERM -exqlite_release(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +exqlite_finalize(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { assert(env); statement_t* statement = NULL; - connection_t* conn = NULL; - if (argc != 2) { + if (argc != 1) { return enif_make_badarg(env); } - if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) { - return make_error_tuple(env, "invalid_connection"); - } - - if (!enif_get_resource(env, argv[1], statement_type, (void**)&statement)) { - return make_error_tuple(env, "invalid_statement"); + if (!enif_get_resource(env, argv[0], statement_type, (void**)&statement)) { + return raise_exception(env, "invalid statement"); } if (statement->statement) { @@ -912,11 +981,6 @@ statement_type_destructor(ErlNifEnv* env, void* arg) sqlite3_finalize(statement->statement); statement->statement = NULL; } - - if (statement->conn) { - enif_release_resource(statement->conn); - statement->conn = NULL; - } } static int @@ -993,11 +1057,11 @@ exqlite_enable_load_extension(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ } if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) { - return make_error_tuple(env, "invalid_connection"); + return raise_exception(env, "invalid connection"); } if (!enif_get_int(env, argv[1], &enable_load_extension_value)) { - return make_error_tuple(env, "invalid_enable_load_extension_value"); + return raise_exception(env, "invalid enable_load_extension value"); } rc = sqlite3_enable_load_extension(conn->db, enable_load_extension_value); @@ -1059,11 +1123,11 @@ exqlite_set_update_hook(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) { - return make_error_tuple(env, "invalid_connection"); + return raise_exception(env, "invalid connection"); } if (!enif_get_local_pid(env, argv[1], &conn->update_hook_pid)) { - return make_error_tuple(env, "invalid_pid"); + return raise_exception(env, "invalid pid"); } // Passing the connection as the third argument causes it to be @@ -1112,7 +1176,7 @@ exqlite_set_log_hook(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ErlNifPid* pid = (ErlNifPid*)enif_alloc(sizeof(ErlNifPid)); if (!enif_get_local_pid(env, argv[0], pid)) { enif_free(pid); - return make_error_tuple(env, "invalid_pid"); + return raise_exception(env, "invalid pid"); } enif_mutex_lock(log_hook_mutex); @@ -1144,7 +1208,7 @@ exqlite_interrupt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) { - return make_error_tuple(env, "invalid_connection"); + return raise_exception(env, "invalid connection"); } // DB is already closed, nothing to do here @@ -1157,29 +1221,36 @@ exqlite_interrupt(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) return make_atom(env, "ok"); } -// -// Most of our nif functions are going to be IO bounded -// - static ErlNifFunc nif_funcs[] = { - {"open", 2, exqlite_open, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"close", 1, exqlite_close, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"execute", 2, exqlite_execute, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"changes", 1, exqlite_changes, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"prepare", 2, exqlite_prepare, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"bind", 3, exqlite_bind, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"step", 2, exqlite_step, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"multi_step", 3, exqlite_multi_step, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"columns", 2, exqlite_columns, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"last_insert_rowid", 1, exqlite_last_insert_rowid, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"transaction_status", 1, exqlite_transaction_status, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"serialize", 2, exqlite_serialize, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"deserialize", 3, exqlite_deserialize, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"release", 2, exqlite_release, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"enable_load_extension", 2, exqlite_enable_load_extension, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"set_update_hook", 2, exqlite_set_update_hook, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"set_log_hook", 1, exqlite_set_log_hook, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"interrupt", 1, exqlite_interrupt, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"dirty_io_open", 2, exqlite_open, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"dirty_io_close", 1, exqlite_close, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"dirty_io_execute", 2, exqlite_execute, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"dirty_io_step", 2, exqlite_step, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"dirty_io_serialize", 2, exqlite_serialize, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"dirty_io_deserialize", 3, exqlite_deserialize, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"dirty_io_interrupt", 1, exqlite_interrupt, ERL_NIF_DIRTY_JOB_IO_BOUND}, + + {"dirty_io_multi_step", 3, exqlite_multi_step, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"dirty_io_insert_all", 3, exqlite_insert_all, ERL_NIF_DIRTY_JOB_IO_BOUND}, + + {"dirty_cpu_prepare", 2, exqlite_prepare, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"dirty_cpu_bind_all", 3, exqlite_bind_all, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + + {"execute", 2, exqlite_execute}, + {"changes", 1, exqlite_changes}, + {"prepare", 2, exqlite_prepare}, + {"columns", 2, exqlite_columns}, + {"step", 2, exqlite_step}, + {"interrupt", 1, exqlite_interrupt}, + {"finalize", 1, exqlite_finalize}, + {"last_insert_rowid", 1, exqlite_last_insert_rowid}, + {"transaction_status", 1, exqlite_transaction_status}, + + {"bind_all", 3, exqlite_bind_all}, + + {"enable_load_extension", 2, exqlite_enable_load_extension}, + {"set_update_hook", 2, exqlite_set_update_hook}, + {"set_log_hook", 1, exqlite_set_log_hook}, }; -ERL_NIF_INIT(Elixir.Exqlite.Sqlite3NIF, nif_funcs, on_load, NULL, NULL, on_unload) +ERL_NIF_INIT(Elixir.Exqlite.Nif, nif_funcs, on_load, NULL, NULL, on_unload) diff --git a/lib/exqlite.ex b/lib/exqlite.ex index af42a7f5..2e81a7fc 100644 --- a/lib/exqlite.ex +++ b/lib/exqlite.ex @@ -3,102 +3,336 @@ defmodule Exqlite do SQLite3 driver for Elixir. """ - alias Exqlite.Connection - alias Exqlite.Error - alias Exqlite.Query - alias Exqlite.Result - - @doc "See `Exqlite.Connection.connect/1`" - @spec start_link([Connection.connection_opt()]) :: {:ok, pid()} | {:error, Error.t()} - def start_link(opts) do - DBConnection.start_link(Connection, opts) - end + alias Exqlite.{Nif, Error} - @spec query(DBConnection.conn(), iodata(), list(), list()) :: - {:ok, Result.t()} | {:error, Exception.t()} - def query(conn, statement, params \\ [], opts \\ []) do - query = %Query{name: "", statement: IO.iodata_to_binary(statement)} + @type db :: reference() + @type stmt :: reference() + @type row :: [binary | number | nil] + @type error :: {:error, Error.t()} - case DBConnection.prepare_execute(conn, query, params, opts) do - {:ok, _query, result} -> - {:ok, result} + # https://www.sqlite.org/c3ref/c_open_autoproxy.html + open_flags = [ + readonly: 0x00000001, + readwrite: 0x00000002, + create: 0x00000004, + deleteonclose: 0x00000008, + exclusive: 0x00000010, + autoproxy: 0x00000020, + uri: 0x00000040, + memory: 0x00000080, + main_db: 0x00000100, + temp_db: 0x00000200, + transient_db: 0x00000400, + main_journal: 0x00000800, + temp_journal: 0x00001000, + subjournal: 0x00002000, + super_journal: 0x00004000, + nomutex: 0x00008000, + fullmutex: 0x00010000, + sharedcache: 0x00020000, + privatecache: 0x00040000, + wal: 0x00080000, + nofollow: 0x01000000, + exrescode: 0x02000000 + ] - otherwise -> - otherwise - end - end + open_flag_names = Enum.map(open_flags, fn {name, _value} -> name end) + open_flag_union = Enum.reduce(open_flag_names, &{:|, [], [&1, &2]}) + @type open_flag :: unquote(open_flag_union) - @spec query!(DBConnection.conn(), iodata(), list(), list()) :: Result.t() - def query!(conn, statement, params \\ [], opts \\ []) do - case query(conn, statement, params, opts) do - {:ok, result} -> result - {:error, err} -> raise err - end + for {name, value} <- open_flags do + defp open_flag(unquote(name)), do: unquote(value) end - @spec prepare(DBConnection.conn(), iodata(), list()) :: - {:ok, Query.t()} | {:error, Exception.t()} - def prepare(conn, name, statement, opts \\ []) do - query = %Query{name: name, statement: statement} - DBConnection.prepare(conn, query, opts) + defp open_flag(invalid) do + raise ArgumentError, "invalid flag: #{inspect(invalid)}" end - @spec prepare!(DBConnection.conn(), iodata(), iodata(), list()) :: Query.t() - def prepare!(conn, name, statement, opts \\ []) do - query = %Query{name: name, statement: statement} - DBConnection.prepare!(conn, query, opts) + @doc """ + Opens a new SQLite database at the path provided. + + - `path` can be `":memory"` to keep the sqlite database in memory + - `flags` are listed in https://www.sqlite.org/c3ref/c_open_autoproxy.html + + The flags parameter must include, at a minimum, one of the following three flag combinations: + + - `:readwrite` and `:create` + - `:readonly` + - `:readwrite` + + Examples: + + # TODO explain + Exqlite.open("test.db", [:readwrite, :create]) + + # TODO explain + Exqlite.open("file://test.db?", [:readonly, :uri]) + + # TODO explain + Exqlite.open(":memory:", [:readwrite, :create, :exrescode]) + + See: https://sqlite.org/c3ref/open.html + """ + @spec open(Path.t(), [open_flag]) :: {:ok, db} | error + def open(path, flags) do + flags = + Enum.reduce(flags, 0, fn flag, acc -> + Bitwise.bor(acc, open_flag(flag)) + end) + + # TODO vfs + wrap_error(Nif.dirty_io_open(path, flags)) end - @spec prepare_execute(DBConnection.conn(), iodata(), iodata(), list(), list()) :: - {:ok, Query.t(), Result.t()} | {:error, Error.t()} - def prepare_execute(conn, name, statement, params, opts \\ []) do - query = %Query{name: name, statement: statement} - DBConnection.prepare_execute(conn, query, params, opts) + @doc """ + Closes the database and releases any underlying resources. + + See: https://sqlite.org/c3ref/close.html + """ + @spec close(db) :: :ok | error + def close(db), do: wrap_error(Nif.dirty_io_close(db)) + + @doc """ + Interrupts a long-running query. + + See: https://sqlite.org/c3ref/interrupt.html + """ + @spec interrupt(db) :: :ok | error + def interrupt(db), do: wrap_error(Nif.interrupt(db)) + + @doc """ + Executes an sql script. Multiple stanzas can be passed at once. + + See: https://sqlite.org/c3ref/exec.html + """ + @spec execute(db, iodata) :: :ok | error + def execute(db, sql), do: wrap_error(Nif.execute(db, sql)) + + @doc """ + Gets the number of changes recently. + + **Note**: If triggers are used, the count may be larger than expected. + + See: https://sqlite.org/c3ref/changes.html + """ + @spec changes(db) :: {:ok, integer} | error + def changes(db), do: wrap_error(Nif.changes(db)) + + # TODO prepare_v3 + + @doc """ + Prepares a statement for execution. + + See: https://sqlite.org/c3ref/prepare.html + """ + @spec prepare(db, iodata) :: {:ok, stmt} | error + def prepare(db, sql), do: wrap_error(Nif.prepare(db, sql)) + + @doc "Same as `prepare/2` but runs on DIRTY IO scheduler." + @spec dirty_cpu_prepare(db, iodata) :: {:ok, stmt} | error + def dirty_cpu_prepare(db, sql), do: wrap_error(Nif.dirty_cpu_prepare(db, sql)) + + @doc """ + Binds the arguments to the prepared statement. + Run on dirty CPU scheduler. + + See: https://www.sqlite.org/c3ref/bind_blob.html + """ + @spec bind_all(db, stmt, [binary | number | nil]) :: :ok | error + def bind_all(conn, statement, args) do + wrap_error(Nif.dirty_cpu_bind_all(conn, statement, args)) end - @spec prepare_execute!(DBConnection.conn(), iodata(), iodata(), list(), list()) :: - {Query.t(), Result.t()} - def prepare_execute!(conn, name, statement, params, opts \\ []) do - query = %Query{name: name, statement: statement} - DBConnection.prepare_execute!(conn, query, params, opts) + @doc "Same as `bind_all/3` but runs on a regular, non-dirty scheduler and might block the VM." + @spec unsafe_bind_all(db, stmt, [binary | number | nil]) :: :ok | error + def unsafe_bind_all(conn, statement, args) do + wrap_error(Nif.bind_all(conn, statement, args)) end - @spec execute(DBConnection.conn(), Query.t(), list(), list()) :: - {:ok, Result.t()} | {:error, Error.t()} - def execute(conn, query, params, opts \\ []) do - DBConnection.execute(conn, query, params, opts) + @doc """ + Returns the columns in the result set. + Runs on a regular, non-dirty scheduler. + + See: + - https://www.sqlite.org/c3ref/column_count.html + - https://www.sqlite.org/c3ref/column_name.html + """ + @spec columns(db, stmt) :: {:ok, [String.t()]} | error + def columns(db, stmt), do: wrap_error(Nif.columns(db, stmt)) + + @doc """ + Executes the prepared statement once. Runs on dirty IO scheduler. + + See: https://sqlite.org/c3ref/step.html + """ + @spec step(db, stmt) :: {:row, row} | :done | error + def step(db, stmt), do: wrap_error(Nif.dirty_io_step(db, stmt)) + + @doc """ + Same as `step/2` but runs on a regular, non-dirty scheduler and might block the VM. + """ + @spec unsafe_step(db, stmt) :: {:row, row} | :done | error + def unsafe_step(db, stmt), do: wrap_error(Nif.step(db, stmt)) + + @doc """ + Returns the rowid of the most recent successful INSERT. + + See: https://sqlite.org/c3ref/last_insert_rowid.html + """ + @spec last_insert_rowid(db) :: pos_integer + def last_insert_rowid(db), do: Nif.last_insert_rowid(db) + + # TODO + @spec transaction_status(db) :: :idle | :transaction + def transaction_status(db), do: Nif.transaction_status(db) + + @doc """ + Serialize the contents of the database to a binary. + + See: https://sqlite.org/c3ref/serialize.html + """ + @spec serialize(db, String.t()) :: {:ok, binary} | error + def serialize(db, schema), do: wrap_error(Nif.dirty_io_serialize(db, schema)) + + @doc """ + Disconnect from database and then reopen as an in-memory database based on + the serialized binary. + + See: https://sqlite.org/c3ref/deserialize.html + """ + @spec deserialize(db, String.t(), binary) :: :ok | error + def deserialize(db, schema, serialized) do + wrap_error(Nif.dirty_io_deserialize(db, schema, serialized)) end - @spec execute!(DBConnection.conn(), Query.t(), list(), list()) :: Result.t() - def execute!(conn, query, params, opts \\ []) do - DBConnection.execute!(conn, query, params, opts) + @doc """ + Once finished with the prepared statement, call this to release the underlying + resources. + + This should be called whenever you are done operating with the prepared statement. If + the system has a high load the garbage collector may not clean up the prepared + statements in a timely manner and causing higher than normal levels of memory + pressure. + + If you are operating on limited memory capacity systems, definitely call this. + + See: https://sqlite.org/c3ref/finalize.html + """ + @spec finalize(stmt) :: :ok | error + def finalize(stmt), do: wrap_error(Nif.finalize(stmt)) + + @doc """ + Allows or disallows loading native extensions. + + See: https://sqlite.org/c3ref/enable_load_extension.html + """ + @spec enable_load_extension(db, boolean) :: :ok | error + def enable_load_extension(db, flag) do + wrap_error( + case flag do + true -> Nif.enable_load_extension(db, 1) + false -> Nif.enable_load_extension(db, 0) + end + ) end - @spec close(DBConnection.conn(), Query.t(), list()) :: :ok | {:error, Exception.t()} - def close(conn, query, opts \\ []) do - with {:ok, _} <- DBConnection.close(conn, query, opts) do - :ok + # TODO + @doc """ + Send data change notifications to a process. + + Each time an insert, update, or delete is performed on the connection provided + as the first argument, a message will be sent to the pid provided as the second argument. + + The message is of the form: `{action, db_name, table, row_id}`, where: + + * `action` is one of `:insert`, `:update` or `:delete` + * `db_name` is a string representing the database name where the change took place + * `table` is a string representing the table name where the change took place + * `row_id` is an integer representing the unique row id assigned by SQLite + + ## Restrictions + + * There are some conditions where the update hook will not be invoked by SQLite. + See the documentation for [more details](https://www.sqlite.org/c3ref/update_hook.html) + + * Only one pid can listen to the changes on a given database connection at a time. + If this function is called multiple times for the same connection, only the last pid will + receive the notifications + + * Updates only happen for the connection that is opened. For example, there + are two connections A and B. When an update happens on connection B, the + hook set for connection A will not receive the update, but the hook for + connection B will receive the update. + + See: https://www.sqlite.org/c3ref/update_hook.html + """ + @spec set_update_hook(db, pid) :: :ok | error + def set_update_hook(db, pid), do: wrap_error(Nif.set_update_hook(db, pid)) + + @doc """ + Send log messages to a process. + + Each time a message is logged in SQLite a message will be sent to the pid provided as the argument. + + The message is of the form: `{:log, rc, message}`, where: + + * `rc` is an integer [result code](https://www.sqlite.org/rescode.html) or an [extended result code](https://www.sqlite.org/rescode.html#extrc) + * `message` is a string representing the log message + + See [`SQLITE_CONFIG_LOG`](https://www.sqlite.org/c3ref/c_config_covering_index_scan.html) and + ["The Error And Warning Log"](https://www.sqlite.org/errlog.html) for more details. + + ## Restrictions + + * Only one pid can listen to the log messages at a time. + If this function is called multiple times, only the last pid will + receive the notifications + + See: https://sqlite.org/errlog.html + """ + @spec set_log_hook(pid) :: :ok | error + def set_log_hook(pid), do: wrap_error(Nif.set_log_hook(pid)) + + @doc """ + Executes the prepared statement multiple times. This is a performance optimization. + """ + @spec multi_step(db, stmt, pos_integer) :: {:rows, [row]} | {:done, [row]} | error + def multi_step(db, stmt, steps) do + case Nif.dirty_io_multi_step(db, stmt, steps) do + # TODO + {:rows, rows} -> {:rows, Enum.reverse(rows)} + {:done, rows} -> {:done, Enum.reverse(rows)} + {:error, _code, _message} = error -> wrap_error(error) end end - @spec close!(DBConnection.conn(), Query.t(), list()) :: :ok - def close!(conn, query, opts \\ []) do - DBConnection.close!(conn, query, opts) - :ok + @doc """ + Fetches all rows from the prepared statement. + """ + @spec fetch_all(db, stmt, pos_integer) :: {:ok, [row]} | error + def fetch_all(db, stmt, steps) do + {:ok, try_fetch_all(db, stmt, steps)} + catch + :throw, {:error, _error} = error -> error end - @spec transaction(DBConnection.conn(), (DBConnection.t() -> result), list()) :: - {:ok, result} | {:error, any} - when result: var - def transaction(conn, fun, opts \\ []) do - DBConnection.transaction(conn, fun, opts) + defp try_fetch_all(db, stmt, steps) do + case multi_step(db, stmt, steps) do + {:done, rows} -> rows + # TODO + {:rows, rows} -> rows ++ try_fetch_all(db, stmt, steps) + {:error, _error} = error -> throw(error) + end end - @spec rollback(DBConnection.t(), term()) :: no_return() - def rollback(conn, reason), do: DBConnection.rollback(conn, reason) + @spec insert_all(db, stmt, [row]) :: :ok | error + def insert_all(db, stmt, rows), + do: wrap_error(Nif.dirty_io_insert_all(db, stmt, rows)) - @spec child_spec([Connection.connection_opt()]) :: :supervisor.child_spec() - def child_spec(opts) do - DBConnection.child_spec(Connection, opts) + # TODO + defp wrap_error({:error, code, message}) do + {:error, Error.exception(code: code, message: message)} end + + defp wrap_error(success), do: success end diff --git a/lib/exqlite/basic.ex b/lib/exqlite/basic.ex deleted file mode 100644 index 00b35d8e..00000000 --- a/lib/exqlite/basic.ex +++ /dev/null @@ -1,48 +0,0 @@ -defmodule Exqlite.Basic do - @moduledoc """ - A very basic API for simple use cases. - """ - - alias Exqlite.Connection - alias Exqlite.Query - alias Exqlite.Sqlite3 - alias Exqlite.Error - alias Exqlite.Result - - def open(path) do - Connection.connect(database: path) - end - - def close(%Connection{} = conn) do - case Sqlite3.close(conn.db) do - :ok -> :ok - {:error, reason} -> {:error, %Error{message: to_string(reason)}} - end - end - - def exec(%Connection{} = conn, stmt, args \\ []) do - %Query{statement: stmt} |> Connection.handle_execute(args, [], conn) - end - - def rows(exec_result) do - case exec_result do - {:ok, %Query{}, %Result{rows: rows, columns: columns}, %Connection{}} -> - {:ok, rows, columns} - - {:error, %Error{message: message}, %Connection{}} -> - {:error, to_string(message)} - end - end - - def load_extension(conn, path) do - exec(conn, "select load_extension(?)", [path]) - end - - def enable_load_extension(conn) do - Sqlite3.enable_load_extension(conn.db, true) - end - - def disable_load_extension(conn) do - Sqlite3.enable_load_extension(conn.db, false) - end -end diff --git a/lib/exqlite/connection.ex b/lib/exqlite/connection.ex deleted file mode 100644 index f774ce40..00000000 --- a/lib/exqlite/connection.ex +++ /dev/null @@ -1,721 +0,0 @@ -defmodule Exqlite.Connection do - @moduledoc """ - This module implements connection details as defined in DBProtocol. - - ## Attributes - - - `db` - The sqlite3 database reference. - - `path` - The path that was used to open. - - `transaction_status` - The status of the connection. Can be `:idle` or `:transaction`. - - ## Unknowns - - - How are pooled connections going to work? Since sqlite3 doesn't allow for - simultaneous access. We would need to check if the write ahead log is - enabled on the database. We can't assume and set the WAL pragma because the - database may be stored on a network volume which would cause potential - issues. - - Notes: - - we try to closely follow structure and naming convention of myxql. - - sqlite thrives when there are many small conventions, so we may not implement - some strategies employed by other adapters. See https://sqlite.org/np1queryprob.html - """ - - use DBConnection - alias Exqlite.Error - alias Exqlite.Pragma - alias Exqlite.Query - alias Exqlite.Result - alias Exqlite.Sqlite3 - require Logger - - defstruct [ - :db, - :directory, - :path, - :transaction_status, - :status, - :chunk_size, - :before_disconnect - ] - - @type t() :: %__MODULE__{ - db: Sqlite3.db(), - directory: String.t() | nil, - path: String.t(), - transaction_status: :idle | :transaction, - status: :idle | :busy, - chunk_size: integer(), - before_disconnect: (t -> any) | {module, atom, [any]} | nil - } - - @type journal_mode() :: :delete | :truncate | :persist | :memory | :wal | :off - @type temp_store() :: :default | :file | :memory - @type synchronous() :: :extra | :full | :normal | :off - @type auto_vacuum() :: :none | :full | :incremental - @type locking_mode() :: :normal | :exclusive - - @type connection_opt() :: - {:database, String.t()} - | {:mode, Sqlite3.open_opt()} - | {:journal_mode, journal_mode()} - | {:temp_store, temp_store()} - | {:synchronous, synchronous()} - | {:foreign_keys, :on | :off} - | {:cache_size, integer()} - | {:cache_spill, :on | :off} - | {:case_sensitive_like, boolean()} - | {:auto_vacuum, auto_vacuum()} - | {:locking_mode, locking_mode()} - | {:secure_delete, :on | :off} - | {:wal_auto_check_point, integer()} - | {:busy_timeout, integer()} - | {:chunk_size, integer()} - | {:journal_size_limit, integer()} - | {:soft_heap_limit, integer()} - | {:hard_heap_limit, integer()} - | {:key, String.t()} - | {:custom_pragmas, [{keyword(), integer() | boolean() | String.t()}]} - | {:before_disconnect, (t -> any) | {module, atom, [any]} | nil} - - @impl true - @doc """ - Initializes the Ecto Exqlite adapter. - - For connection configurations we use the defaults that come with SQLite3, but - we recommend which options to choose. We do not default to the recommended - because we don't know what your environment is like. - - Allowed options: - - * `:database` - The path to the database. In memory is allowed. You can use - `:memory` or `":memory:"` to designate that. - * `:mode` - use `:readwrite` to open the database for reading and writing - , `:readonly` to open it in read-only mode or `[:readonly | :readwrite, :nomutex]` - to open it with no mutex mode. `:readwrite` will also create - the database if it doesn't already exist. Defaults to `:readwrite`. - Note: [:readwrite, :nomutex] is not recommended. - * `:journal_mode` - Sets the journal mode for the sqlite connection. Can be - one of the following `:delete`, `:truncate`, `:persist`, `:memory`, - `:wal`, or `:off`. Defaults to `:delete`. It is recommended that you use - `:wal` due to support for concurrent reads. Note: `:wal` does not mean - concurrent writes. - * `:temp_store` - Sets the storage used for temporary tables. Default is - `:default`. Allowed values are `:default`, `:file`, `:memory`. It is - recommended that you use `:memory` for storage. - * `:synchronous` - Can be `:extra`, `:full`, `:normal`, or `:off`. Defaults - to `:normal`. - * `:foreign_keys` - Sets if foreign key checks should be enforced or not. - Can be `:on` or `:off`. Default is `:on`. - * `:cache_size` - Sets the cache size to be used for the connection. This is - an odd setting as a positive value is the number of pages in memory to use - and a negative value is the size in kilobytes to use. Default is `-2000`. - It is recommended that you use `-64000`. - * `:cache_spill` - The cache_spill pragma enables or disables the ability of - the pager to spill dirty cache pages to the database file in the middle of - a transaction. By default it is `:on`, and for most applications, it - should remain so. - * `:case_sensitive_like` - * `:auto_vacuum` - Defaults to `:none`. Can be `:none`, `:full` or - `:incremental`. Depending on the database size, `:incremental` may be - beneficial. - * `:locking_mode` - Defaults to `:normal`. Allowed values are `:normal` or - `:exclusive`. See [sqlite documentation][1] for more information. - * `:secure_delete` - Defaults to `:off`. If enabled, it will cause SQLite3 - to overwrite records that were deleted with zeros. - * `:wal_auto_check_point` - Sets the write-ahead log auto-checkpoint - interval. Default is `1000`. Setting the auto-checkpoint size to zero or a - negative value turns auto-checkpointing off. - * `:busy_timeout` - Sets the busy timeout in milliseconds for a connection. - Default is `2000`. - * `:chunk_size` - The chunk size for bulk fetching. Defaults to `50`. - * `:key` - Optional key to set during database initialization. This PRAGMA - is often used to set up database level encryption. - * `:journal_size_limit` - The size limit in bytes of the journal. - * `:soft_heap_limit` - The size limit in bytes for the heap limit. - * `:hard_heap_limit` - The size limit in bytes for the heap. - * `:custom_pragmas` - A list of custom pragmas to set on the connection, for example to configure extensions. - * `:load_extensions` - A list of paths identifying extensions to load. Defaults to `[]`. - The provided list will be merged with the global extensions list, set on `:exqlite, :load_extensions`. - Be aware that the path should handle pointing to a library compiled for the current architecture. - Example configuration: - - ``` - arch_dir = - System.cmd("uname", ["-sm"]) - |> elem(0) - |> String.trim() - |> String.replace(" ", "-") - |> String.downcase() # => "darwin-arm64" - - config :myapp, arch_dir: arch_dir - - # global - config :exqlite, load_extensions: [ "./priv/sqlite/\#{arch_dir}/rotate" ] - - # per connection in a Phoenix app - config :myapp, Myapp.Repo, - database: "path/to/db", - load_extensions: [ - "./priv/sqlite/\#{arch_dir}/vector0", - "./priv/sqlite/\#{arch_dir}/vss0" - ] - ``` - * `:before_disconnect` - A function to run before disconnect, either a - 2-arity fun or `{module, function, args}` with the close reason and - `t:Exqlite.Connection.t/0` prepended to `args` or `nil` (default: `nil`) - - For more information about the options above, see [sqlite documentation][1] - - [1]: https://www.sqlite.org/pragma.html - """ - @spec connect([connection_opt()]) :: {:ok, t()} | {:error, Exception.t()} - def connect(options) do - database = Keyword.get(options, :database) - - options = - Keyword.put_new( - options, - :chunk_size, - Application.get_env(:exqlite, :default_chunk_size, 50) - ) - - case database do - nil -> - {:error, - %Error{ - message: """ - You must provide a :database to the database. \ - Example: connect(database: "./") or connect(database: :memory)\ - """ - }} - - :memory -> - do_connect(":memory:", options) - - _ -> - do_connect(database, options) - end - end - - @impl true - def disconnect(err, %__MODULE__{db: db} = state) do - if state.before_disconnect != nil do - apply(state.before_disconnect, [err, state]) - end - - case Sqlite3.close(db) do - :ok -> :ok - {:error, reason} -> {:error, %Error{message: to_string(reason)}} - end - end - - @impl true - def checkout(%__MODULE__{status: :idle} = state) do - {:ok, %{state | status: :busy}} - end - - def checkout(%__MODULE__{status: :busy} = state) do - {:disconnect, %Error{message: "Database is busy"}, state} - end - - @impl true - def ping(state), do: {:ok, state} - - ## - ## Handlers - ## - - @impl true - def handle_prepare(%Query{} = query, options, state) do - with {:ok, query} <- prepare(query, options, state) do - {:ok, query, state} - end - end - - @impl true - def handle_execute(%Query{} = query, params, options, state) do - with {:ok, query} <- prepare(query, options, state) do - execute(:execute, query, params, state) - end - end - - @doc """ - Begin a transaction. - - For full info refer to sqlite docs: https://sqlite.org/lang_transaction.html - - Note: default transaction mode is DEFERRED. - """ - @impl true - def handle_begin(options, %{transaction_status: transaction_status} = state) do - # This doesn't handle more than 2 levels of transactions. - # - # One possible solution would be to just track the number of open - # transactions and use that for driving the transaction status being idle or - # in a transaction. - # - # I do not know why the other official adapters do not track this and just - # append level on the savepoint. Instead the rollbacks would just completely - # revert the issues when it may be desirable to fix something while in the - # transaction and then commit. - case Keyword.get(options, :mode, :deferred) do - :deferred when transaction_status == :idle -> - handle_transaction(:begin, "BEGIN TRANSACTION", state) - - :transaction when transaction_status == :idle -> - handle_transaction(:begin, "BEGIN TRANSACTION", state) - - :immediate when transaction_status == :idle -> - handle_transaction(:begin, "BEGIN IMMEDIATE TRANSACTION", state) - - :exclusive when transaction_status == :idle -> - handle_transaction(:begin, "BEGIN EXCLUSIVE TRANSACTION", state) - - mode - when mode in [:deferred, :immediate, :exclusive, :savepoint] and - transaction_status == :transaction -> - handle_transaction(:begin, "SAVEPOINT exqlite_savepoint", state) - end - end - - @impl true - def handle_commit(options, %{transaction_status: transaction_status} = state) do - case Keyword.get(options, :mode, :deferred) do - :savepoint when transaction_status == :transaction -> - handle_transaction( - :commit_savepoint, - "RELEASE SAVEPOINT exqlite_savepoint", - state - ) - - mode - when mode in [:deferred, :immediate, :exclusive, :transaction] and - transaction_status == :transaction -> - handle_transaction(:commit, "COMMIT", state) - end - end - - @impl true - def handle_rollback(options, %{transaction_status: transaction_status} = state) do - case Keyword.get(options, :mode, :deferred) do - :savepoint when transaction_status == :transaction -> - with {:ok, _result, state} <- - handle_transaction( - :rollback_savepoint, - "ROLLBACK TO SAVEPOINT exqlite_savepoint", - state - ) do - handle_transaction( - :rollback_savepoint, - "RELEASE SAVEPOINT exqlite_savepoint", - state - ) - end - - mode - when mode in [:deferred, :immediate, :exclusive, :transaction] -> - handle_transaction(:rollback, "ROLLBACK TRANSACTION", state) - end - end - - @doc """ - Close a query prepared by `handle_prepare/3` with the database. Return - `{:ok, result, state}` on success and to continue, - `{:error, exception, state}` to return an error and continue, or - `{:disconnect, exception, state}` to return an error and disconnect. - - This callback is called in the client process. - """ - @impl true - def handle_close(query, _opts, state) do - Sqlite3.release(state.db, query.ref) - {:ok, nil, state} - end - - @impl true - def handle_declare(%Query{} = query, params, opts, state) do - # We emulate cursor functionality by just using a prepared statement and - # step through it. Thus we just return the query ref as the cursor. - with {:ok, query} <- prepare_no_cache(query, opts, state), - {:ok, query} <- bind_params(query, params, state) do - {:ok, query, query.ref, state} - end - end - - @impl true - def handle_deallocate(%Query{} = query, _cursor, _opts, state) do - Sqlite3.release(state.db, query.ref) - {:ok, nil, state} - end - - @impl true - def handle_fetch(%Query{statement: statement}, cursor, opts, state) do - chunk_size = opts[:chunk_size] || opts[:max_rows] || state.chunk_size - - case Sqlite3.multi_step(state.db, cursor, chunk_size) do - {:done, rows} -> - {:halt, %Result{rows: rows, command: :fetch, num_rows: length(rows)}, state} - - {:rows, rows} -> - {:cont, %Result{rows: rows, command: :fetch, num_rows: chunk_size}, state} - - {:error, reason} -> - {:error, %Error{message: to_string(reason), statement: statement}, state} - - :busy -> - {:error, %Error{message: "Database is busy", statement: statement}, state} - end - end - - @impl true - def handle_status(_opts, state) do - {state.transaction_status, state} - end - - ### ---------------------------------- - # Internal functions and helpers - ### ---------------------------------- - - defp set_pragma(db, pragma_name, value) do - Sqlite3.execute(db, "PRAGMA #{pragma_name} = #{value}") - end - - defp get_pragma(db, pragma_name) do - {:ok, statement} = Sqlite3.prepare(db, "PRAGMA #{pragma_name}") - - case Sqlite3.fetch_all(db, statement) do - {:ok, [[value]]} -> {:ok, value} - _ -> :error - end - end - - defp maybe_set_pragma(db, pragma_name, value) do - case get_pragma(db, pragma_name) do - {:ok, current} -> - if current == value do - :ok - else - set_pragma(db, pragma_name, value) - end - - _ -> - set_pragma(db, pragma_name, value) - end - end - - defp set_key(db, options) do - # we can't use maybe_set_pragma here since - # the only thing that will work on an encrypted - # database without error is setting the key. - case Keyword.fetch(options, :key) do - {:ok, key} -> set_pragma(db, "key", key) - _ -> :ok - end - end - - defp set_custom_pragmas(db, options) do - # we can't use maybe_set_pragma because some pragmas - # are required to be set before the database is e.g. decrypted. - case Keyword.fetch(options, :custom_pragmas) do - {:ok, list} -> do_set_custom_pragmas(db, list) - _ -> :ok - end - end - - defp do_set_custom_pragmas(db, list) do - list - |> Enum.reduce_while(:ok, fn {key, value}, :ok -> - case set_pragma(db, key, value) do - :ok -> {:cont, :ok} - {:error, _reason} -> {:halt, :error} - end - end) - end - - defp set_pragma_if_present(_db, _pragma, nil), do: :ok - defp set_pragma_if_present(db, pragma, value), do: set_pragma(db, pragma, value) - - defp set_journal_size_limit(db, options) do - set_pragma_if_present( - db, - "journal_size_limit", - Keyword.get(options, :journal_size_limit) - ) - end - - defp set_soft_heap_limit(db, options) do - set_pragma_if_present(db, "soft_heap_limit", Keyword.get(options, :soft_heap_limit)) - end - - defp set_hard_heap_limit(db, options) do - set_pragma_if_present(db, "hard_heap_limit", Keyword.get(options, :hard_heap_limit)) - end - - defp set_journal_mode(db, options) do - maybe_set_pragma(db, "journal_mode", Pragma.journal_mode(options)) - end - - defp set_temp_store(db, options) do - set_pragma(db, "temp_store", Pragma.temp_store(options)) - end - - defp set_synchronous(db, options) do - set_pragma(db, "synchronous", Pragma.synchronous(options)) - end - - defp set_foreign_keys(db, options) do - set_pragma(db, "foreign_keys", Pragma.foreign_keys(options)) - end - - defp set_cache_size(db, options) do - maybe_set_pragma(db, "cache_size", Pragma.cache_size(options)) - end - - defp set_cache_spill(db, options) do - set_pragma(db, "cache_spill", Pragma.cache_spill(options)) - end - - defp set_case_sensitive_like(db, options) do - set_pragma(db, "case_sensitive_like", Pragma.case_sensitive_like(options)) - end - - defp set_auto_vacuum(db, options) do - set_pragma(db, "auto_vacuum", Pragma.auto_vacuum(options)) - end - - defp set_locking_mode(db, options) do - set_pragma(db, "locking_mode", Pragma.locking_mode(options)) - end - - defp set_secure_delete(db, options) do - set_pragma(db, "secure_delete", Pragma.secure_delete(options)) - end - - defp set_wal_auto_check_point(db, options) do - set_pragma(db, "wal_autocheckpoint", Pragma.wal_auto_check_point(options)) - end - - defp set_busy_timeout(db, options) do - set_pragma(db, "busy_timeout", Pragma.busy_timeout(options)) - end - - defp load_extensions(db, options) do - global_extensions = Application.get_env(:exqlite, :load_extensions, []) - - extensions = - Keyword.get(options, :load_extensions, []) - |> Enum.concat(global_extensions) - |> Enum.uniq() - - do_load_extensions(db, extensions) - end - - defp do_load_extensions(_db, []), do: :ok - - defp do_load_extensions(db, extensions) do - Sqlite3.enable_load_extension(db, true) - - Enum.each(extensions, fn extension -> - Logger.debug(fn -> "Exqlite: loading extension `#{extension}`" end) - Sqlite3.execute(db, "SELECT load_extension('#{extension}')") - end) - - Sqlite3.enable_load_extension(db, false) - end - - defp do_connect(database, options) do - with {:ok, directory} <- resolve_directory(database), - :ok <- mkdir_p(directory), - {:ok, db} <- Sqlite3.open(database, options), - :ok <- set_key(db, options), - :ok <- set_custom_pragmas(db, options), - :ok <- set_journal_mode(db, options), - :ok <- set_temp_store(db, options), - :ok <- set_synchronous(db, options), - :ok <- set_foreign_keys(db, options), - :ok <- set_cache_size(db, options), - :ok <- set_cache_spill(db, options), - :ok <- set_auto_vacuum(db, options), - :ok <- set_locking_mode(db, options), - :ok <- set_secure_delete(db, options), - :ok <- set_wal_auto_check_point(db, options), - :ok <- set_case_sensitive_like(db, options), - :ok <- set_busy_timeout(db, options), - :ok <- set_journal_size_limit(db, options), - :ok <- set_soft_heap_limit(db, options), - :ok <- set_hard_heap_limit(db, options), - :ok <- load_extensions(db, options) do - state = %__MODULE__{ - db: db, - directory: directory, - path: database, - transaction_status: :idle, - status: :idle, - chunk_size: Keyword.get(options, :chunk_size), - before_disconnect: Keyword.get(options, :before_disconnect, nil) - } - - {:ok, state} - else - {:error, reason} -> - {:error, %Exqlite.Error{message: to_string(reason)}} - end - end - - def maybe_put_command(query, options) do - case Keyword.get(options, :command) do - nil -> query - command -> %{query | command: command} - end - end - - # Attempt to retrieve the cached query, if it doesn't exist, we'll prepare one - # and cache it for later. - defp prepare(%Query{statement: statement} = query, options, state) do - query = maybe_put_command(query, options) - - with {:ok, ref} <- Sqlite3.prepare(state.db, IO.iodata_to_binary(statement)), - query <- %{query | ref: ref} do - {:ok, query} - else - {:error, reason} -> - {:error, %Error{message: to_string(reason), statement: statement}, state} - end - end - - # Prepare a query and do not cache it. - defp prepare_no_cache(%Query{statement: statement} = query, options, state) do - query = maybe_put_command(query, options) - - case Sqlite3.prepare(state.db, statement) do - {:ok, ref} -> - {:ok, %{query | ref: ref}} - - {:error, reason} -> - {:error, %Error{message: to_string(reason), statement: statement}, state} - end - end - - @spec maybe_changes(Sqlite3.db(), Query.t()) :: integer() | nil - defp maybe_changes(db, %Query{command: command}) - when command in [:update, :insert, :delete] do - case Sqlite3.changes(db) do - {:ok, total} -> total - _ -> nil - end - end - - defp maybe_changes(_, _), do: nil - - # when we have an empty list of columns, that signifies that - # there was no possible return tuple (e.g., update statement without RETURNING) - # and in that case, we return nil to signify no possible result. - defp maybe_rows([], []), do: nil - defp maybe_rows(rows, _cols), do: rows - - defp execute(call, %Query{} = query, params, state) do - with {:ok, query} <- bind_params(query, params, state), - {:ok, columns} <- get_columns(query, state), - {:ok, rows} <- get_rows(query, state), - {:ok, transaction_status} <- Sqlite3.transaction_status(state.db), - changes <- maybe_changes(state.db, query) do - case query.command do - command when command in [:delete, :insert, :update] -> - { - :ok, - query, - Result.new( - command: call, - num_rows: changes, - rows: maybe_rows(rows, columns) - ), - %{state | transaction_status: transaction_status} - } - - _ -> - { - :ok, - query, - Result.new( - command: call, - columns: columns, - rows: rows, - num_rows: Enum.count(rows) - ), - %{state | transaction_status: transaction_status} - } - end - end - end - - defp bind_params(%Query{ref: ref, statement: statement} = query, params, state) - when ref != nil do - case Sqlite3.bind(state.db, ref, params) do - :ok -> - {:ok, query} - - {:error, reason} -> - {:error, %Error{message: to_string(reason), statement: statement}, state} - end - end - - defp get_columns(%Query{ref: ref, statement: statement}, state) do - case Sqlite3.columns(state.db, ref) do - {:ok, columns} -> - {:ok, columns} - - {:error, reason} -> - {:error, %Error{message: to_string(reason), statement: statement}, state} - end - end - - defp get_rows(%Query{ref: ref, statement: statement}, state) do - case Sqlite3.fetch_all(state.db, ref, state.chunk_size) do - {:ok, rows} -> - {:ok, rows} - - {:error, reason} -> - {:error, %Error{message: to_string(reason), statement: statement}, state} - end - end - - defp handle_transaction(call, statement, state) do - with :ok <- Sqlite3.execute(state.db, statement), - {:ok, transaction_status} <- Sqlite3.transaction_status(state.db) do - result = %Result{ - command: call, - rows: [], - columns: [], - num_rows: 0 - } - - {:ok, result, %{state | transaction_status: transaction_status}} - else - {:error, reason} -> - {:disconnect, %Error{message: to_string(reason), statement: statement}, state} - end - end - - defp resolve_directory(":memory:"), do: {:ok, nil} - - defp resolve_directory("file:" <> _ = uri) do - case URI.parse(uri) do - %{path: path} when is_binary(path) -> - {:ok, Path.dirname(path)} - - _ -> - {:error, "No path in #{inspect(uri)}"} - end - end - - defp resolve_directory(path), do: {:ok, Path.dirname(path)} - - # SQLITE_OPEN_CREATE will create the DB file if not existing, but - # will not create intermediary directories if they are missing. - # So let's preemptively create the intermediate directories here - # before trying to open the DB file. - defp mkdir_p(nil), do: :ok - defp mkdir_p(directory), do: File.mkdir_p(directory) -end diff --git a/lib/exqlite/error.ex b/lib/exqlite/error.ex index d445e354..b2c296c3 100644 --- a/lib/exqlite/error.ex +++ b/lib/exqlite/error.ex @@ -1,18 +1,21 @@ defmodule Exqlite.Error do @moduledoc """ - The error emitted from SQLite or a general error with the library. - """ + Wraps an SQLite3 error. + + See: https://sqlite.org/rescode.html - defexception [:message, :statement] + Examples: - @type t :: %__MODULE__{ - message: String.t(), - statement: String.t() - } + # SQLITE_MISUSE + # https://sqlite.org/rescode.html#misuse + %Exqlite.Error{code: 21, message: "TODO"} - @impl true - def message(%__MODULE__{message: message, statement: nil}), do: message + # SQLITE_BUSY + # https://sqlite.org/rescode.html#busy + %Exqlite.Error{code: 5, message: "TODO"} - def message(%__MODULE__{message: message, statement: statement}), - do: "#{message}\n#{statement}" + """ + @enforce_keys [:code, :message] + defexception [:code, :message] + @type t :: %__MODULE__{code: pos_integer, message: String.t()} end diff --git a/lib/exqlite/flags.ex b/lib/exqlite/flags.ex deleted file mode 100644 index 537700f3..00000000 --- a/lib/exqlite/flags.ex +++ /dev/null @@ -1,35 +0,0 @@ -defmodule Exqlite.Flags do - @moduledoc false - - import Bitwise - - # https://www.sqlite.org/c3ref/c_open_autoproxy.html - @file_open_flags [ - sqlite_open_readonly: 0x00000001, - sqlite_open_readwrite: 0x00000002, - sqlite_open_create: 0x00000004, - sqlite_open_deleteonclos: 0x00000008, - sqlite_open_exclusive: 0x00000010, - sqlite_open_autoproxy: 0x00000020, - sqlite_open_uri: 0x00000040, - sqlite_open_memory: 0x00000080, - sqlite_open_main_db: 0x00000100, - sqlite_open_temp_db: 0x00000200, - sqlite_open_transient_db: 0x00000400, - sqlite_open_main_journal: 0x00000800, - sqlite_open_temp_journal: 0x00001000, - sqlite_open_subjournal: 0x00002000, - sqlite_open_super_journal: 0x00004000, - sqlite_open_nomutex: 0x00008000, - sqlite_open_fullmutex: 0x00010000, - sqlite_open_sharedcache: 0x00020000, - sqlite_open_privatecache: 0x00040000, - sqlite_open_wal: 0x00080000, - sqlite_open_nofollow: 0x01000000, - sqlite_open_exrescode: 0x02000000 - ] - - def put_file_open_flags(current_flags \\ 0, flags) do - Enum.reduce(flags, current_flags, &(&2 ||| Keyword.fetch!(@file_open_flags, &1))) - end -end diff --git a/lib/exqlite/nif.ex b/lib/exqlite/nif.ex new file mode 100644 index 00000000..8d2c1f20 --- /dev/null +++ b/lib/exqlite/nif.ex @@ -0,0 +1,43 @@ +defmodule Exqlite.Nif do + @moduledoc false + @compile {:autoload, false} + @on_load {:load_nif, 0} + + def load_nif do + path = :filename.join(:code.priv_dir(:exqlite), ~c"sqlite3_nif") + :erlang.load_nif(path, 0) + end + + # TODO individual binds + + def dirty_io_open(_path, _flags), do: :erlang.nif_error(:not_loaded) + def dirty_io_close(_db), do: :erlang.nif_error(:not_loaded) + def dirty_io_execute(_db, _sql), do: :erlang.nif_error(:not_loaded) + def dirty_io_step(_db, _stmt), do: :erlang.nif_error(:not_loaded) + def dirty_io_serialize(_db, _schema), do: :erlang.nif_error(:not_loaded) + def dirty_io_deserialize(_db, _schema, _buffer), do: :erlang.nif_error(:not_loaded) + def dirty_io_interrupt(_db), do: :erlang.nif_error(:not_loaded) + + def dirty_io_multi_step(_db, _stmt, _steps), do: :erlang.nif_error(:not_loaded) + def dirty_io_insert_all(_db, _stmt, _rows), do: :erlang.nif_error(:not_loaded) + + def dirty_cpu_prepare(_db, _sql), do: :erlang.nif_error(:not_loaded) + def dirty_cpu_bind_all(_db, _stmt, _params), do: :erlang.nif_error(:not_loaded) + + def execute(_db, _sql), do: :erlang.nif_error(:not_loaded) + def changes(_db), do: :erlang.nif_error(:not_loaded) + def prepare(_db, _sql), do: :erlang.nif_error(:not_loaded) + def columns(_db, _stmt), do: :erlang.nif_error(:not_loaded) + + def step(_db, _stmt), do: :erlang.nif_error(:not_loaded) + def interrupt(_db), do: :erlang.nif_error(:not_loaded) + def finalize(_stmt), do: :erlang.nif_error(:not_loaded) + def last_insert_rowid(_db), do: :erlang.nif_error(:not_loaded) + def transaction_status(_db), do: :erlang.nif_error(:not_loaded) + + def bind_all(_db, _stmt, _params), do: :erlang.nif_error(:not_loaded) + + def enable_load_extension(_db, _path), do: :erlang.nif_error(:not_loaded) + def set_update_hook(_db, _pid), do: :erlang.nif_error(:not_loaded) + def set_log_hook(_pid), do: :erlang.nif_error(:not_loaded) +end diff --git a/lib/exqlite/pragma.ex b/lib/exqlite/pragma.ex deleted file mode 100644 index 52c518c9..00000000 --- a/lib/exqlite/pragma.ex +++ /dev/null @@ -1,121 +0,0 @@ -defmodule Exqlite.Pragma do - @moduledoc """ - Handles parsing extra options for the SQLite connection - """ - - def busy_timeout(nil), do: busy_timeout([]) - - def busy_timeout(options) do - Keyword.get(options, :busy_timeout, 2000) - end - - def journal_mode(nil), do: journal_mode([]) - - def journal_mode(options) do - case Keyword.get(options, :journal_mode, :delete) do - :delete -> "delete" - :memory -> "memory" - :off -> "off" - :persist -> "persist" - :truncate -> "truncate" - :wal -> "wal" - _ -> raise ArgumentError, "invalid :journal_mode" - end - end - - def temp_store(nil), do: temp_store([]) - - def temp_store(options) do - case Keyword.get(options, :temp_store, :default) do - :file -> 1 - :memory -> 2 - :default -> 0 - _ -> raise ArgumentError, "invalid :temp_store" - end - end - - def synchronous(nil), do: synchronous([]) - - def synchronous(options) do - case Keyword.get(options, :synchronous, :normal) do - :extra -> 3 - :full -> 2 - :normal -> 1 - :off -> 0 - _ -> raise ArgumentError, "invalid :synchronous" - end - end - - def foreign_keys(nil), do: foreign_keys([]) - - def foreign_keys(options) do - case Keyword.get(options, :foreign_keys, :on) do - :off -> 0 - :on -> 1 - _ -> raise ArgumentError, "invalid :foreign_keys" - end - end - - def cache_size(nil), do: cache_size([]) - - def cache_size(options) do - Keyword.get(options, :cache_size, -2000) - end - - def cache_spill(nil), do: cache_spill([]) - - def cache_spill(options) do - case Keyword.get(options, :cache_spill, :on) do - :off -> 0 - :on -> 1 - _ -> raise ArgumentError, "invalid :cache_spill" - end - end - - def case_sensitive_like(nil), do: case_sensitive_like([]) - - def case_sensitive_like(options) do - case Keyword.get(options, :case_sensitive_like, :off) do - :off -> 0 - :on -> 1 - _ -> raise ArgumentError, "invalid :case_sensitive_like" - end - end - - def auto_vacuum(nil), do: auto_vacuum([]) - - def auto_vacuum(options) do - case Keyword.get(options, :auto_vacuum, :none) do - :none -> 0 - :full -> 1 - :incremental -> 2 - _ -> raise ArgumentError, "invalid :auto_vacuum" - end - end - - def locking_mode(nil), do: locking_mode([]) - - def locking_mode(options) do - case Keyword.get(options, :locking_mode, :normal) do - :normal -> "NORMAL" - :exclusive -> "EXCLUSIVE" - _ -> raise ArgumentError, "invalid :locking_mode" - end - end - - def secure_delete(nil), do: secure_delete([]) - - def secure_delete(options) do - case Keyword.get(options, :secure_delete, :off) do - :off -> 0 - :on -> 1 - _ -> raise ArgumentError, "invalid :secure_delete" - end - end - - def wal_auto_check_point(nil), do: wal_auto_check_point([]) - - def wal_auto_check_point(options) do - Keyword.get(options, :wal_auto_check_point, 1000) - end -end diff --git a/lib/exqlite/query.ex b/lib/exqlite/query.ex deleted file mode 100644 index 5b12e14d..00000000 --- a/lib/exqlite/query.ex +++ /dev/null @@ -1,70 +0,0 @@ -defmodule Exqlite.Query do - @moduledoc """ - Query struct returned from a successfully prepared query. - """ - @type t :: %__MODULE__{ - statement: iodata(), - name: atom() | String.t(), - ref: reference() | nil, - command: :insert | :delete | :update | nil - } - - defstruct statement: nil, - name: nil, - ref: nil, - command: nil - - def build(options) do - statement = Keyword.get(options, :statement) - name = Keyword.get(options, :name) - ref = Keyword.get(options, :ref) - - command = - case Keyword.get(options, :command) do - nil -> extract_command(statement) - value -> value - end - - %__MODULE__{ - statement: statement, - name: name, - ref: ref, - command: command - } - end - - defp extract_command(nil), do: nil - - defp extract_command(statement) do - cond do - String.contains?(statement, "INSERT") -> :insert - String.contains?(statement, "DELETE") -> :delete - String.contains?(statement, "UPDATE") -> :update - true -> nil - end - end - - defimpl DBConnection.Query do - def parse(query, _opts) do - query - end - - def describe(query, _opts) do - query - end - - def encode(_query, params, _opts) do - params - end - - def decode(_query, result, _opts) do - result - end - end - - defimpl String.Chars do - def to_string(%{statement: statement}) do - IO.iodata_to_binary(statement) - end - end -end diff --git a/lib/exqlite/result.ex b/lib/exqlite/result.ex deleted file mode 100644 index 53f59391..00000000 --- a/lib/exqlite/result.ex +++ /dev/null @@ -1,35 +0,0 @@ -defmodule Exqlite.Result do - @moduledoc """ - The database results. - """ - - @type t :: %__MODULE__{ - command: atom, - columns: [String.t()] | nil, - rows: [[term] | term] | nil, - num_rows: integer() - } - - defstruct command: nil, columns: [], rows: [], num_rows: 0 - - def new(options) do - %__MODULE__{ - command: Keyword.get(options, :command), - columns: Keyword.get(options, :columns, []), - rows: Keyword.get(options, :rows, []), - num_rows: Keyword.get(options, :num_rows, 0) - } - end -end - -if Code.ensure_loaded?(Table.Reader) do - defimpl Table.Reader, for: Exqlite.Result do - def init(%{columns: columns}) when columns in [nil, []] do - {:rows, %{columns: []}, []} - end - - def init(result) do - {:rows, %{columns: result.columns}, result.rows} - end - end -end diff --git a/lib/exqlite/sqlite3.ex b/lib/exqlite/sqlite3.ex deleted file mode 100644 index 1f25044c..00000000 --- a/lib/exqlite/sqlite3.ex +++ /dev/null @@ -1,305 +0,0 @@ -defmodule Exqlite.Sqlite3 do - @moduledoc """ - The interface to the NIF implementation. - """ - - # If the database reference is closed, any prepared statements should be - # dereferenced as well. It is entirely possible that an application does - # not properly remove a stale reference. - # - # Will need to add a test for this and think of possible solution. - - # Need to figure out if we can just stream results where we use this - # module as a sink. - - alias Exqlite.Flags - alias Exqlite.Sqlite3NIF - - @type db() :: reference() - @type statement() :: reference() - @type reason() :: atom() | String.t() - @type row() :: list() - @type open_mode :: :readwrite | :readonly | :nomutex - @type open_opt :: {:mode, :readwrite | :readonly | [open_mode()]} - - @doc """ - Opens a new sqlite database at the Path provided. - - `path` can be `":memory"` to keep the sqlite database in memory. - - ## Options - - * `:mode` - use `:readwrite` to open the database for reading and writing - , `:readonly` to open it in read-only mode or `[:readonly | :readwrite, :nomutex]` - to open it with no mutex mode. `:readwrite` will also create - the database if it doesn't already exist. Defaults to `:readwrite`. - Note: [:readwrite, :nomutex] is not recommended. - """ - @spec open(String.t(), [open_opt()]) :: {:ok, db()} | {:error, reason()} - def open(path, opts \\ []) do - mode = Keyword.get(opts, :mode, :readwrite) - Sqlite3NIF.open(path, flags_from_mode(mode)) - end - - defp flags_from_mode(:nomutex) do - raise ArgumentError, - "expected mode to be `:readwrite` or `:readonly`, can't use a single :nomutex mode" - end - - defp flags_from_mode(:readwrite), - do: do_flags_from_mode([:readwrite], []) - - defp flags_from_mode(:readonly), - do: do_flags_from_mode([:readonly], []) - - defp flags_from_mode([_ | _] = modes), - do: do_flags_from_mode(modes, []) - - defp flags_from_mode(mode) do - raise ArgumentError, - "expected mode to be `:readwrite`, `:readonly` or list of modes, but received #{inspect(mode)}" - end - - defp do_flags_from_mode([:readwrite | tail], acc), - do: do_flags_from_mode(tail, [:sqlite_open_readwrite, :sqlite_open_create | acc]) - - defp do_flags_from_mode([:readonly | tail], acc), - do: do_flags_from_mode(tail, [:sqlite_open_readonly | acc]) - - defp do_flags_from_mode([:nomutex | tail], acc), - do: do_flags_from_mode(tail, [:sqlite_open_nomutex | acc]) - - defp do_flags_from_mode([mode | _tail], _acc) do - raise ArgumentError, - "expected mode to be `:readwrite`, `:readonly` or `:nomutex`, but received #{inspect(mode)}" - end - - defp do_flags_from_mode([], acc), - do: Flags.put_file_open_flags(acc) - - @doc """ - Closes the database and releases any underlying resources. - """ - @spec close(db() | nil) :: :ok | {:error, reason()} - def close(nil), do: :ok - def close(conn), do: Sqlite3NIF.close(conn) - - @doc """ - Interrupt a long-running query. - """ - @spec interrupt(db() | nil) :: :ok | {:error, reason()} - def interrupt(nil), do: :ok - def interrupt(conn), do: Sqlite3NIF.interrupt(conn) - - @doc """ - Executes an sql script. Multiple stanzas can be passed at once. - """ - @spec execute(db(), String.t()) :: :ok | {:error, reason()} - def execute(conn, sql), do: Sqlite3NIF.execute(conn, sql) - - @doc """ - Get the number of changes recently. - - **Note**: If triggers are used, the count may be larger than expected. - - See: https://sqlite.org/c3ref/changes.html - """ - @spec changes(db()) :: {:ok, integer()} | {:error, reason()} - def changes(conn), do: Sqlite3NIF.changes(conn) - - @spec prepare(db(), String.t()) :: {:ok, statement()} | {:error, reason()} - def prepare(conn, sql), do: Sqlite3NIF.prepare(conn, sql) - - @spec bind(db(), statement(), nil) :: :ok | {:error, reason()} - def bind(conn, statement, nil), do: bind(conn, statement, []) - - @spec bind(db(), statement(), list()) :: :ok | {:error, reason()} - def bind(conn, statement, args) do - Sqlite3NIF.bind(conn, statement, Enum.map(args, &convert/1)) - end - - @spec columns(db(), statement()) :: {:ok, [binary()]} | {:error, reason()} - def columns(conn, statement), do: Sqlite3NIF.columns(conn, statement) - - @spec step(db(), statement()) :: :done | :busy | {:row, row()} | {:error, reason()} - def step(conn, statement), do: Sqlite3NIF.step(conn, statement) - - @spec multi_step(db(), statement()) :: - :busy | {:rows, [row()]} | {:done, [row()]} | {:error, reason()} - def multi_step(conn, statement) do - chunk_size = Application.get_env(:exqlite, :default_chunk_size, 50) - multi_step(conn, statement, chunk_size) - end - - @spec multi_step(db(), statement(), integer()) :: - :busy | {:rows, [row()]} | {:done, [row()]} | {:error, reason()} - def multi_step(conn, statement, chunk_size) do - case Sqlite3NIF.multi_step(conn, statement, chunk_size) do - :busy -> - :busy - - {:error, reason} -> - {:error, reason} - - {:rows, rows} -> - {:rows, Enum.reverse(rows)} - - {:done, rows} -> - {:done, Enum.reverse(rows)} - end - end - - @spec last_insert_rowid(db()) :: {:ok, integer()} - def last_insert_rowid(conn), do: Sqlite3NIF.last_insert_rowid(conn) - - @spec transaction_status(db()) :: {:ok, :idle | :transaction} - def transaction_status(conn), do: Sqlite3NIF.transaction_status(conn) - - @doc """ - Causes the database connection to free as much memory as it can. This is - useful if you are on a memory restricted system. - """ - @spec shrink_memory(db()) :: :ok | {:error, reason()} - def shrink_memory(conn) do - Sqlite3NIF.execute(conn, "PRAGMA shrink_memory") - end - - @spec fetch_all(db(), statement(), integer()) :: {:ok, [row()]} | {:error, reason()} - def fetch_all(conn, statement, chunk_size) do - {:ok, try_fetch_all(conn, statement, chunk_size)} - catch - :throw, {:error, _reason} = error -> error - end - - defp try_fetch_all(conn, statement, chunk_size) do - case multi_step(conn, statement, chunk_size) do - {:done, rows} -> rows - {:rows, rows} -> rows ++ try_fetch_all(conn, statement, chunk_size) - {:error, _reason} = error -> throw(error) - :busy -> throw({:error, "Database busy"}) - end - end - - @spec fetch_all(db(), statement()) :: {:ok, [row()]} | {:error, reason()} - def fetch_all(conn, statement) do - # Should this be done in the NIF? It can be _much_ faster to build a list - # there, but at the expense that it could block other dirty nifs from - # getting work done. - # - # For now this just works - chunk_size = Application.get_env(:exqlite, :default_chunk_size, 50) - fetch_all(conn, statement, chunk_size) - end - - @doc """ - Serialize the contents of the database to a binary. - """ - @spec serialize(db(), String.t()) :: {:ok, binary()} | {:error, reason()} - def serialize(conn, database \\ "main") do - Sqlite3NIF.serialize(conn, database) - end - - @doc """ - Disconnect from database and then reopen as an in-memory database based on - the serialized binary. - """ - @spec deserialize(db(), String.t(), binary()) :: :ok | {:error, reason()} - def deserialize(conn, database \\ "main", serialized) do - Sqlite3NIF.deserialize(conn, database, serialized) - end - - def release(_conn, nil), do: :ok - - @doc """ - Once finished with the prepared statement, call this to release the underlying - resources. - - This should be called whenever you are done operating with the prepared statement. If - the system has a high load the garbage collector may not clean up the prepared - statements in a timely manner and causing higher than normal levels of memory - pressure. - - If you are operating on limited memory capacity systems, definitely call this. - """ - @spec release(db(), statement()) :: :ok | {:error, reason()} - def release(conn, statement) do - Sqlite3NIF.release(conn, statement) - end - - @doc """ - Allow loading native extensions. - """ - @spec enable_load_extension(db(), boolean()) :: :ok | {:error, reason()} - def enable_load_extension(conn, flag) do - if flag do - Sqlite3NIF.enable_load_extension(conn, 1) - else - Sqlite3NIF.enable_load_extension(conn, 0) - end - end - - @doc """ - Send data change notifications to a process. - - Each time an insert, update, or delete is performed on the connection provided - as the first argument, a message will be sent to the pid provided as the second argument. - - The message is of the form: `{action, db_name, table, row_id}`, where: - - * `action` is one of `:insert`, `:update` or `:delete` - * `db_name` is a string representing the database name where the change took place - * `table` is a string representing the table name where the change took place - * `row_id` is an integer representing the unique row id assigned by SQLite - - ## Restrictions - - * There are some conditions where the update hook will not be invoked by SQLite. - See the documentation for [more details](https://www.sqlite.org/c3ref/update_hook.html) - * Only one pid can listen to the changes on a given database connection at a time. - If this function is called multiple times for the same connection, only the last pid will - receive the notifications - * Updates only happen for the connection that is opened. For example, there - are two connections A and B. When an update happens on connection B, the - hook set for connection A will not receive the update, but the hook for - connection B will receive the update. - """ - @spec set_update_hook(db(), pid()) :: :ok | {:error, reason()} - def set_update_hook(conn, pid) do - Sqlite3NIF.set_update_hook(conn, pid) - end - - @doc """ - Send log messages to a process. - - Each time a message is logged in SQLite a message will be sent to the pid provided as the argument. - - The message is of the form: `{:log, rc, message}`, where: - - * `rc` is an integer [result code](https://www.sqlite.org/rescode.html) or an [extended result code](https://www.sqlite.org/rescode.html#extrc) - * `message` is a string representing the log message - - See [`SQLITE_CONFIG_LOG`](https://www.sqlite.org/c3ref/c_config_covering_index_scan.html) and - ["The Error And Warning Log"](https://www.sqlite.org/errlog.html) for more details. - - ## Restrictions - - * Only one pid can listen to the log messages at a time. - If this function is called multiple times, only the last pid will - receive the notifications - """ - @spec set_log_hook(pid()) :: :ok | {:error, reason()} - def set_log_hook(pid) do - Sqlite3NIF.set_log_hook(pid) - end - - defp convert(%Date{} = val), do: Date.to_iso8601(val) - defp convert(%Time{} = val), do: Time.to_iso8601(val) - defp convert(%NaiveDateTime{} = val), do: NaiveDateTime.to_iso8601(val) - defp convert(%DateTime{time_zone: "Etc/UTC"} = val), do: NaiveDateTime.to_iso8601(val) - - defp convert(%DateTime{} = datetime) do - raise ArgumentError, "#{inspect(datetime)} is not in UTC" - end - - defp convert(val), do: val -end diff --git a/lib/exqlite/sqlite3_nif.ex b/lib/exqlite/sqlite3_nif.ex deleted file mode 100644 index 356daca6..00000000 --- a/lib/exqlite/sqlite3_nif.ex +++ /dev/null @@ -1,77 +0,0 @@ -defmodule Exqlite.Sqlite3NIF do - @moduledoc """ - This is the module where all of the NIF entry points reside. Calling this directly - should be avoided unless you are aware of what you are doing. - """ - - @compile {:autoload, false} - @on_load {:load_nif, 0} - - @type db() :: reference() - @type statement() :: reference() - @type reason() :: :atom | String.Chars.t() - @type row() :: list() - - def load_nif() do - path = :filename.join(:code.priv_dir(:exqlite), ~c"sqlite3_nif") - :erlang.load_nif(path, 0) - end - - @spec open(String.t(), integer()) :: {:ok, db()} | {:error, reason()} - def open(_path, _flags), do: :erlang.nif_error(:not_loaded) - - @spec close(db()) :: :ok | {:error, reason()} - def close(_conn), do: :erlang.nif_error(:not_loaded) - - @spec interrupt(db()) :: :ok | {:error, reason()} - def interrupt(_conn), do: :erlang.nif_error(:not_loaded) - - @spec execute(db(), String.t()) :: :ok | {:error, reason()} - def execute(_conn, _sql), do: :erlang.nif_error(:not_loaded) - - @spec changes(db()) :: {:ok, integer()} | {:error, reason()} - def changes(_conn), do: :erlang.nif_error(:not_loaded) - - @spec prepare(db(), String.t()) :: {:ok, statement()} | {:error, reason()} - def prepare(_conn, _sql), do: :erlang.nif_error(:not_loaded) - - @spec bind(db(), statement(), list()) :: - :ok | {:error, reason()} | {:error, {atom(), any()}} - def bind(_conn, _statement, _args), do: :erlang.nif_error(:not_loaded) - - @spec step(db(), statement()) :: :done | :busy | {:row, row()} | {:error, reason()} - def step(_conn, _statement), do: :erlang.nif_error(:not_loaded) - - @spec multi_step(db(), statement(), integer()) :: - :busy | {:rows, [row()]} | {:done, [row()]} | {:error, reason()} - def multi_step(_conn, _statement, _chunk_size), do: :erlang.nif_error(:not_loaded) - - @spec columns(db(), statement()) :: {:ok, list(binary())} | {:error, reason()} - def columns(_conn, _statement), do: :erlang.nif_error(:not_loaded) - - @spec last_insert_rowid(db()) :: {:ok, integer()} - def last_insert_rowid(_conn), do: :erlang.nif_error(:not_loaded) - - @spec transaction_status(db()) :: {:ok, :idle | :transaction} - def transaction_status(_conn), do: :erlang.nif_error(:not_loaded) - - @spec serialize(db(), String.t()) :: {:ok, binary()} | {:error, reason()} - def serialize(_conn, _database), do: :erlang.nif_error(:not_loaded) - - @spec deserialize(db(), String.t(), binary()) :: :ok | {:error, reason()} - def deserialize(_conn, _database, _serialized), do: :erlang.nif_error(:not_loaded) - - @spec release(db(), statement()) :: :ok | {:error, reason()} - def release(_conn, _statement), do: :erlang.nif_error(:not_loaded) - - @spec enable_load_extension(db(), integer()) :: :ok | {:error, reason()} - def enable_load_extension(_conn, _flag), do: :erlang.nif_error(:not_loaded) - - @spec set_update_hook(db(), pid()) :: :ok | {:error, reason()} - def set_update_hook(_conn, _pid), do: :erlang.nif_error(:not_loaded) - - @spec set_log_hook(pid()) :: :ok | {:error, reason()} - def set_log_hook(_pid), do: :erlang.nif_error(:not_loaded) - - # add statement inspection tooling https://sqlite.org/c3ref/expanded_sql.html -end diff --git a/lib/exqlite/stream.ex b/lib/exqlite/stream.ex deleted file mode 100644 index 79ff5d86..00000000 --- a/lib/exqlite/stream.ex +++ /dev/null @@ -1,42 +0,0 @@ -defmodule Exqlite.Stream do - @moduledoc false - defstruct [:conn, :query, :params, :options] - @type t :: %Exqlite.Stream{} - - defimpl Enumerable do - def reduce(%Exqlite.Stream{query: %Exqlite.Query{} = query} = stream, acc, fun) do - # Possibly need to pass a chunk size option along so that we can let - # the NIF chunk it. - %Exqlite.Stream{conn: conn, params: params, options: opts} = stream - - stream = %DBConnection.Stream{ - conn: conn, - query: query, - params: params, - opts: opts - } - - DBConnection.reduce(stream, acc, fun) - end - - def reduce(%Exqlite.Stream{query: statement} = stream, acc, fun) do - %Exqlite.Stream{conn: conn, params: params, options: opts} = stream - query = %Exqlite.Query{name: "", statement: statement} - - stream = %DBConnection.PrepareStream{ - conn: conn, - query: query, - params: params, - opts: opts - } - - DBConnection.reduce(stream, acc, fun) - end - - def member?(_, _), do: {:error, __MODULE__} - - def count(_), do: {:error, __MODULE__} - - def slice(_), do: {:error, __MODULE__} - end -end diff --git a/lib/mix/tasks/test_sqlite_version.ex b/lib/mix/tasks/test_sqlite_version.ex index 325e424d..37e08eb4 100644 --- a/lib/mix/tasks/test_sqlite_version.ex +++ b/lib/mix/tasks/test_sqlite_version.ex @@ -9,9 +9,8 @@ defmodule Mix.Tasks.TestSqliteVersion do mix_version = Exqlite.MixProject.sqlite_version() # Get installed SQLite version - {:ok, conn} = Exqlite.Sqlite3.open(":memory:") - {:ok, stmt} = Exqlite.Sqlite3.prepare(conn, "select sqlite_version()") - {_, [amalgamation_version]} = Exqlite.Sqlite3.step(conn, stmt) + {:ok, [[amalgamation_version]]} = + with_db(fn db -> all(db, "select sqlite_version()") end) if mix_version != amalgamation_version do ("mix test_sqlite_version failed: the mix.exs version (#{mix_version}) " <> @@ -19,4 +18,24 @@ defmodule Mix.Tasks.TestSqliteVersion do |> Mix.raise() end end + + defp with_db(f) do + with {:ok, db} <- Exqlite.open(":memory:", [:readonly]) do + try do + f.(db) + after + Exqlite.close(db) + end + end + end + + defp all(db, sql) do + with {:ok, stmt} <- Exqlite.prepare(db, sql) do + try do + Exqlite.fetch_all(db, stmt, _steps = 100) + after + Exqlite.finalize(stmt) + end + end + end end diff --git a/mix.exs b/mix.exs index 48b77bd0..87a130d1 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Exqlite.MixProject do use Mix.Project - @version "0.24.1" + @version "1.0.0-rc.0" @sqlite_version "3.46.1" def project do @@ -25,9 +25,16 @@ defmodule Exqlite.MixProject do deps: deps(), package: package(), description: description(), - test_paths: test_paths(System.get_env("EXQLITE_INTEGRATION")), elixirc_paths: elixirc_paths(Mix.env()), dialyzer: dialyzer(), + test_coverage: [ + # TODO + ignore_modules: [ + Exqlite.Nif, + Mix.Tasks.DownloadSqlite, + Mix.Tasks.TestSqliteVersion + ] + ], # Docs name: "Exqlite", @@ -49,15 +56,13 @@ defmodule Exqlite.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do [ - {:db_connection, "~> 2.1"}, {:ex_sqlean, "~> 0.8.5", only: [:dev, :test]}, {:elixir_make, "~> 0.8", runtime: false}, {:cc_precompiler, "~> 0.1", runtime: false}, {:ex_doc, "~> 0.27", only: :dev, runtime: false}, - {:temp, "~> 0.4", only: [:dev, :test]}, {:credo, "~> 1.6", only: [:dev, :test], runtime: false}, {:dialyxir, "~> 1.3.0", only: [:dev, :test], runtime: false}, - {:table, "~> 0.1.0", optional: true} + {:benchee, "~> 1.3", only: :bench} ] end @@ -121,9 +126,6 @@ defmodule Exqlite.MixProject do defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] - defp test_paths(nil), do: ["test"] - defp test_paths(_any), do: ["integration_test/exqlite"] - defp dialyzer do [ plt_add_deps: :apps_direct, diff --git a/mix.lock b/mix.lock index c777feef..4476c1b3 100644 --- a/mix.lock +++ b/mix.lock @@ -1,8 +1,9 @@ %{ + "benchee": {:hex, :benchee, "1.3.1", "c786e6a76321121a44229dde3988fc772bca73ea75170a73fd5f4ddf1af95ccf", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "76224c58ea1d0391c8309a8ecbfe27d71062878f59bd41a390266bf4ac1cc56d"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "cc_precompiler": {:hex, :cc_precompiler, "0.1.10", "47c9c08d8869cf09b41da36538f62bc1abd3e19e41701c2cea2675b53c704258", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "f6e046254e53cd6b41c6bacd70ae728011aa82b2742a80d6e2214855c6e06b22"}, "credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"}, - "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, + "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "dialyxir": {:hex, :dialyxir, "1.3.0", "fd1672f0922b7648ff9ce7b1b26fcf0ef56dda964a459892ad15f6b4410b5284", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "00b2a4bcd6aa8db9dcb0b38c1225b7277dca9bc370b6438715667071a304696f"}, "earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"}, "elixir_make": {:hex, :elixir_make, "0.8.4", "4960a03ce79081dee8fe119d80ad372c4e7badb84c493cc75983f9d3bc8bde0f", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "6e7f1d619b5f61dfabd0a20aa268e575572b542ac31723293a4c1a567d5ef040"}, @@ -15,7 +16,5 @@ "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.1", "c7f58c120b2b5aa5fd80d540a89fdf866ed42f1f3994e4fe189abebeab610839", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "8a89a1eeccc2d798d6ea15496a6e4870b75e014d1af514b1b71fa33134f57814"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, - "table": {:hex, :table, "0.1.2", "87ad1125f5b70c5dea0307aa633194083eb5182ec537efc94e96af08937e14a8", [:mix], [], "hexpm", "7e99bc7efef806315c7e65640724bf165c3061cdc5d854060f74468367065029"}, - "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, - "temp": {:hex, :temp, "0.4.9", "eb6355bfa7925a568b3d9eb3bb57e89aa6d2b78bfe8dfb6b698e090631b7f41f", [:mix], [], "hexpm", "bc8bf7b27d9105bef933ef4bf4ba37ac6b899dbeba329deaa88c60b62d6b4b6d"}, + "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"}, } diff --git a/test/exqlite/connection_test.exs b/test/exqlite/connection_test.exs index 5264cdab..264b9bfc 100644 --- a/test/exqlite/connection_test.exs +++ b/test/exqlite/connection_test.exs @@ -1,391 +1,202 @@ -defmodule Exqlite.ConnectionTest do - use ExUnit.Case +# defmodule Exqlite.ConnectionTest do +# use ExUnit.Case, async: true - alias Exqlite.Connection - alias Exqlite.Query - alias Exqlite.Sqlite3 +# describe ".disconnect/2" do +# test "disconnects a database that was never connected" do +# conn = %Connection{db: nil, path: nil} - describe ".connect/1" do - test "returns error when path is missing from options" do - {:error, error} = Connection.connect([]) +# assert :ok == Connection.disconnect(nil, conn) +# end - assert error.message == - ~s{You must provide a :database to the database. Example: connect(database: "./") or connect(database: :memory)} - end +# test "disconnects a connected database" do +# {:ok, conn} = Connection.connect(database: :memory) - test "connects to an in memory database" do - {:ok, state} = Connection.connect(database: ":memory:") +# assert :ok == Connection.disconnect(nil, conn) +# end - assert state.path == ":memory:" - assert state.db - end +# test "executes before_disconnect before disconnecting" do +# {:ok, pid} = Agent.start_link(fn -> 0 end) - test "connects to in memory when the memory atom is passed" do - {:ok, state} = Connection.connect(database: :memory) +# {:ok, conn} = +# Connection.connect( +# database: :memory, +# before_disconnect: fn err, db -> +# Agent.update(pid, fn count -> count + 1 end) +# assert err == true +# assert db +# end +# ) - assert state.path == ":memory:" - assert state.db - end +# assert :ok == Connection.disconnect(true, conn) +# assert Agent.get(pid, &Function.identity/1) == 1 +# end +# end - test "connects to a file" do - path = Temp.path!() - {:ok, state} = Connection.connect(database: path) +# describe ".handle_execute/4" do +# test "returns records" do +# path = Temp.path!() - assert state.path == path - assert state.db +# {:ok, db} = Sqlite3.open(path) - File.rm(path) - end +# :ok = +# Sqlite3.execute(db, "create table users (id integer primary key, name text)") - test "connects to a file with an accented character" do - path = Temp.path!(prefix: "databasΓ©") - {:ok, state} = Connection.connect(database: path) +# :ok = Sqlite3.execute(db, "insert into users (id, name) values (1, 'Jim')") +# :ok = Sqlite3.execute(db, "insert into users (id, name) values (2, 'Bob')") +# :ok = Sqlite3.execute(db, "insert into users (id, name) values (3, 'Dave')") +# :ok = Sqlite3.execute(db, "insert into users (id, name) values (4, 'Steve')") +# Sqlite3.close(db) - assert state.path == path - assert state.db +# {:ok, conn} = Connection.connect(database: path) - File.rm(path) - end +# {:ok, _query, result, _conn} = +# %Query{statement: "select * from users where id < ?"} +# |> Connection.handle_execute([4], [], conn) - test "connects to a file from URL" do - path = Temp.path!() +# assert result.command == :execute +# assert result.columns == ["id", "name"] +# assert result.rows == [[1, "Jim"], [2, "Bob"], [3, "Dave"]] - {:ok, state} = Connection.connect(database: "file:#{path}?mode=rwc") +# File.rm(path) +# end - assert state.directory == Path.dirname(path) - assert state.db - end +# test "returns correctly for empty result" do +# path = Temp.path!() - test "fails to write a file from URL with mode=ro" do - path = Temp.path!() +# {:ok, db} = Sqlite3.open(path) - {:ok, db} = Sqlite3.open(path) +# :ok = +# Sqlite3.execute(db, "create table users (id integer primary key, name text)") - :ok = - Sqlite3.execute(db, "create table test (id ingeger primary key, stuff text)") +# Sqlite3.close(db) - :ok = - Sqlite3.execute(db, "insert into test (id, stuff) values (999, 'Some stuff')") +# {:ok, conn} = Connection.connect(database: path) - :ok = Sqlite3.close(db) +# {:ok, _query, result, _conn} = +# %Query{ +# statement: "UPDATE users set name = 'wow' where id = 1", +# command: :update +# } +# |> Connection.handle_execute([], [], conn) - {:ok, conn} = Connection.connect(database: "file:#{path}?mode=ro") +# assert result.rows == nil - assert conn.directory == Path.dirname(path) - assert conn.db +# {:ok, _query, result, _conn} = +# %Query{ +# statement: "UPDATE users set name = 'wow' where id = 5 returning *", +# command: :update +# } +# |> Connection.handle_execute([], [], conn) - assert match?( - {:ok, _, %{rows: [[1]]}, _}, - %Query{statement: "select count(*) from test"} - |> Connection.handle_execute([], [], conn) - ) +# assert result.rows == [] - {:error, %{message: message}, _} = - %Query{ - statement: "insert into test (id, stuff) values (888, 'some more stuff')" - } - |> Connection.handle_execute([], [], conn) +# File.rm(path) +# end - # In most of the test matrix the message is "attempt to write a readonly database", - # but in Elixir 1.13, OTP 23, OS windows-2019 it is "not an error". - assert message in ["attempt to write a readonly database", "not an error"] +# test "returns timely and in order for big data sets" do +# path = Temp.path!() - File.rm(path) - end +# {:ok, db} = Sqlite3.open(path) - test "setting custom_pragmas" do - path = Temp.path!() +# :ok = +# Sqlite3.execute(db, "create table users (id integer primary key, name text)") - {:ok, state} = - Connection.connect( - database: path, - custom_pragmas: [ - checkpoint_fullfsync: 0 - ] - ) +# users = +# Enum.map(1..10_000, fn i -> +# [i, "User-#{i}"] +# end) - assert state.db +# users +# |> Enum.chunk_every(20) +# |> Enum.each(fn chunk -> +# values = Enum.map_join(chunk, ", ", fn [id, name] -> "(#{id}, '#{name}')" end) +# Sqlite3.execute(db, "insert into users (id, name) values #{values}") +# end) - assert {:ok, 0} = get_pragma(state.db, :checkpoint_fullfsync) +# :ok = Exqlite.Sqlite3.close(db) - File.rm(path) - end +# {:ok, conn} = Connection.connect(database: path) - test "setting journal_size_limit" do - path = Temp.path!() - size_limit = 20 * 1024 * 1024 - {:ok, state} = Connection.connect(database: path, journal_size_limit: size_limit) +# {:ok, _query, result, _conn} = +# Connection.handle_execute( +# %Exqlite.Query{ +# statement: "SELECT * FROM users" +# }, +# [], +# [timeout: 1], +# conn +# ) - assert state.db +# assert result.command == :execute +# assert length(result.rows) == 10_000 +# assert users == result.rows - assert {:ok, ^size_limit} = get_pragma(state.db, :journal_size_limit) +# File.rm(path) +# end +# end - File.rm(path) - end +# describe ".handle_prepare/3" do +# test "returns a prepared query" do +# {:ok, conn} = Connection.connect(database: :memory) - test "setting soft_heap_limit" do - path = Temp.path!() - size_limit = 20 * 1024 * 1024 - {:ok, state} = Connection.connect(database: path, soft_heap_limit: size_limit) +# {:ok, _query, _result, conn} = +# %Query{statement: "create table users (id integer primary key, name text)"} +# |> Connection.handle_execute([], [], conn) - assert state.db +# {:ok, query, conn} = +# %Query{statement: "select * from users where id < ?"} +# |> Connection.handle_prepare([], conn) - assert {:ok, ^size_limit} = get_pragma(state.db, :soft_heap_limit) +# assert conn +# assert query +# assert query.ref +# assert query.statement +# end - File.rm(path) - end +# test "users table does not exist" do +# {:ok, conn} = Connection.connect(database: :memory) - test "setting hard_heap_limit" do - path = Temp.path!() - size_limit = 20 * 1024 * 1024 - {:ok, state} = Connection.connect(database: path, hard_heap_limit: size_limit) +# {:error, error, _state} = +# %Query{statement: "select * from users where id < ?"} +# |> Connection.handle_prepare([], conn) - assert state.db +# assert error.message == "no such table: users" +# end +# end - assert {:ok, ^size_limit} = get_pragma(state.db, :hard_heap_limit) +# describe ".checkout/1" do +# test "checking out an idle connection" do +# {:ok, conn} = Connection.connect(database: :memory) - File.rm(path) - end +# {:ok, conn} = Connection.checkout(conn) +# assert conn.status == :busy +# end - test "setting connection mode" do - path = Temp.path!() +# test "checking out a busy connection" do +# {:ok, conn} = Connection.connect(database: :memory) +# conn = %{conn | status: :busy} - # Create readwrite connection - {:ok, rw_state} = Connection.connect(database: path) - create_table_query = "create table test (id integer primary key, stuff text)" - :ok = Sqlite3.execute(rw_state.db, create_table_query) +# {:disconnect, error, _conn} = Connection.checkout(conn) - insert_value_query = "insert into test (stuff) values ('This is a test')" - :ok = Sqlite3.execute(rw_state.db, insert_value_query) +# assert error.message == "Database is busy" +# end +# end - # Read from database with a readonly connection - {:ok, ro_state} = Connection.connect(database: path, mode: :readonly) +# describe ".handle_close/3" do +# test "releases the underlying prepared statement" do +# {:ok, conn} = Connection.connect(database: :memory) - select_query = "select id, stuff from test order by id asc" - {:ok, statement} = Sqlite3.prepare(ro_state.db, select_query) - {:row, columns} = Sqlite3.step(ro_state.db, statement) +# {:ok, query, _result, conn} = +# %Query{statement: "create table users (id integer primary key, name text)"} +# |> Connection.handle_execute([], [], conn) - assert [1, "This is a test"] == columns +# assert {:ok, nil, conn} == Connection.handle_close(query, [], conn) - # Readonly connection cannot insert - assert {:error, "attempt to write a readonly database"} == - Sqlite3.execute(ro_state.db, insert_value_query) - end - end +# {:ok, query, conn} = +# %Query{statement: "select * from users where id < ?"} +# |> Connection.handle_prepare([], conn) - defp get_pragma(db, pragma_name) do - {:ok, statement} = Sqlite3.prepare(db, "PRAGMA #{pragma_name}") - - case Sqlite3.fetch_all(db, statement) do - {:ok, [[value]]} -> {:ok, value} - _ -> :error - end - end - - describe ".disconnect/2" do - test "disconnects a database that was never connected" do - conn = %Connection{db: nil, path: nil} - - assert :ok == Connection.disconnect(nil, conn) - end - - test "disconnects a connected database" do - {:ok, conn} = Connection.connect(database: :memory) - - assert :ok == Connection.disconnect(nil, conn) - end - - test "executes before_disconnect before disconnecting" do - {:ok, pid} = Agent.start_link(fn -> 0 end) - - {:ok, conn} = - Connection.connect( - database: :memory, - before_disconnect: fn err, db -> - Agent.update(pid, fn count -> count + 1 end) - assert err == true - assert db - end - ) - - assert :ok == Connection.disconnect(true, conn) - assert Agent.get(pid, &Function.identity/1) == 1 - end - end - - describe ".handle_execute/4" do - test "returns records" do - path = Temp.path!() - - {:ok, db} = Sqlite3.open(path) - - :ok = - Sqlite3.execute(db, "create table users (id integer primary key, name text)") - - :ok = Sqlite3.execute(db, "insert into users (id, name) values (1, 'Jim')") - :ok = Sqlite3.execute(db, "insert into users (id, name) values (2, 'Bob')") - :ok = Sqlite3.execute(db, "insert into users (id, name) values (3, 'Dave')") - :ok = Sqlite3.execute(db, "insert into users (id, name) values (4, 'Steve')") - Sqlite3.close(db) - - {:ok, conn} = Connection.connect(database: path) - - {:ok, _query, result, _conn} = - %Query{statement: "select * from users where id < ?"} - |> Connection.handle_execute([4], [], conn) - - assert result.command == :execute - assert result.columns == ["id", "name"] - assert result.rows == [[1, "Jim"], [2, "Bob"], [3, "Dave"]] - - File.rm(path) - end - - test "returns correctly for empty result" do - path = Temp.path!() - - {:ok, db} = Sqlite3.open(path) - - :ok = - Sqlite3.execute(db, "create table users (id integer primary key, name text)") - - Sqlite3.close(db) - - {:ok, conn} = Connection.connect(database: path) - - {:ok, _query, result, _conn} = - %Query{ - statement: "UPDATE users set name = 'wow' where id = 1", - command: :update - } - |> Connection.handle_execute([], [], conn) - - assert result.rows == nil - - {:ok, _query, result, _conn} = - %Query{ - statement: "UPDATE users set name = 'wow' where id = 5 returning *", - command: :update - } - |> Connection.handle_execute([], [], conn) - - assert result.rows == [] - - File.rm(path) - end - - test "returns timely and in order for big data sets" do - path = Temp.path!() - - {:ok, db} = Sqlite3.open(path) - - :ok = - Sqlite3.execute(db, "create table users (id integer primary key, name text)") - - users = - Enum.map(1..10_000, fn i -> - [i, "User-#{i}"] - end) - - users - |> Enum.chunk_every(20) - |> Enum.each(fn chunk -> - values = Enum.map_join(chunk, ", ", fn [id, name] -> "(#{id}, '#{name}')" end) - Sqlite3.execute(db, "insert into users (id, name) values #{values}") - end) - - :ok = Exqlite.Sqlite3.close(db) - - {:ok, conn} = Connection.connect(database: path) - - {:ok, _query, result, _conn} = - Connection.handle_execute( - %Exqlite.Query{ - statement: "SELECT * FROM users" - }, - [], - [timeout: 1], - conn - ) - - assert result.command == :execute - assert length(result.rows) == 10_000 - assert users == result.rows - - File.rm(path) - end - end - - describe ".handle_prepare/3" do - test "returns a prepared query" do - {:ok, conn} = Connection.connect(database: :memory) - - {:ok, _query, _result, conn} = - %Query{statement: "create table users (id integer primary key, name text)"} - |> Connection.handle_execute([], [], conn) - - {:ok, query, conn} = - %Query{statement: "select * from users where id < ?"} - |> Connection.handle_prepare([], conn) - - assert conn - assert query - assert query.ref - assert query.statement - end - - test "users table does not exist" do - {:ok, conn} = Connection.connect(database: :memory) - - {:error, error, _state} = - %Query{statement: "select * from users where id < ?"} - |> Connection.handle_prepare([], conn) - - assert error.message == "no such table: users" - end - end - - describe ".checkout/1" do - test "checking out an idle connection" do - {:ok, conn} = Connection.connect(database: :memory) - - {:ok, conn} = Connection.checkout(conn) - assert conn.status == :busy - end - - test "checking out a busy connection" do - {:ok, conn} = Connection.connect(database: :memory) - conn = %{conn | status: :busy} - - {:disconnect, error, _conn} = Connection.checkout(conn) - - assert error.message == "Database is busy" - end - end - - describe ".ping/1" do - test "returns the state passed unchanged" do - {:ok, conn} = Connection.connect(database: :memory) - - assert {:ok, conn} == Connection.ping(conn) - end - end - - describe ".handle_close/3" do - test "releases the underlying prepared statement" do - {:ok, conn} = Connection.connect(database: :memory) - - {:ok, query, _result, conn} = - %Query{statement: "create table users (id integer primary key, name text)"} - |> Connection.handle_execute([], [], conn) - - assert {:ok, nil, conn} == Connection.handle_close(query, [], conn) - - {:ok, query, conn} = - %Query{statement: "select * from users where id < ?"} - |> Connection.handle_prepare([], conn) - - assert {:ok, nil, conn} == Connection.handle_close(query, [], conn) - end - end -end +# assert {:ok, nil, conn} == Connection.handle_close(query, [], conn) +# end +# end +# end diff --git a/test/exqlite/error_test.exs b/test/exqlite/error_test.exs deleted file mode 100644 index efa7ba27..00000000 --- a/test/exqlite/error_test.exs +++ /dev/null @@ -1,15 +0,0 @@ -defmodule Exqlite.ErrorTest do - use ExUnit.Case - - alias Exqlite.Error - - describe "message/1" do - test "with :statement" do - assert "a\nb" == Exception.message(%Error{message: "a", statement: "b"}) - end - - test "without :statement" do - assert "a" == Exception.message(%Error{message: "a"}) - end - end -end diff --git a/test/exqlite/exqlite_test.exs b/test/exqlite/exqlite_test.exs new file mode 100644 index 00000000..5425f441 --- /dev/null +++ b/test/exqlite/exqlite_test.exs @@ -0,0 +1,561 @@ +defmodule ExqliteTest do + use ExUnit.Case, case: :async + + defp all(db, sql, args \\ []) do + with {:ok, stmt} <- Exqlite.prepare(db, sql) do + try do + with :ok <- Exqlite.bind_all(db, stmt, args) do + Exqlite.fetch_all(db, stmt, 100) + end + after + Exqlite.finalize(stmt) + end + end + end + + describe ".open/2" do + test "opens a database in memory" do + assert {:ok, db} = Exqlite.open(":memory:", [:readonly]) + on_exit(fn -> Exqlite.close(db) end) + assert is_reference(db) + end + + @tag :tmp_dir + test "opens a database on disk", %{tmp_dir: tmp_dir} do + path = Path.join(tmp_dir, "db.sqlite") + + assert {:ok, db} = Exqlite.open(path, [:create, :readwrite]) + on_exit(fn -> Exqlite.close(db) end) + assert is_reference(db) + end + + @tag :tmp_dir + test "creates database path on disk when non-existent", %{tmp_dir: tmp_dir} do + path = Path.join(tmp_dir, "db.sqlite") + if File.exists?(path), do: File.rm!(path) + + assert {:ok, db} = Exqlite.open(path, [:create, :readwrite]) + on_exit(fn -> Exqlite.close(db) end) + + assert is_reference(db) + assert File.exists?(path) + end + + @tag :tmp_dir + test "connects to a file from URL", %{tmp_dir: tmp_dir} do + path = Path.join(tmp_dir, "db.sqlite") + + assert {:ok, db} = + Exqlite.open("file:#{path}?mode=rwc", [:uri, :create, :readwrite]) + + on_exit(fn -> Exqlite.close(db) end) + assert is_reference(db) + end + + @tag :tmp_dir + test "opens a database in readonly mode", %{tmp_dir: tmp_dir} do + path = Path.join(tmp_dir, "db.sqlite") + if File.exists?(path), do: File.rm!(path) + + # Create database with readwrite flag + {:ok, rw} = Exqlite.open(path, [:create, :readwrite]) + on_exit(fn -> Exqlite.close(rw) end) + + :ok = Exqlite.execute(rw, "create table test (stuff text)") + :ok = Exqlite.execute(rw, "insert into test (stuff) values ('This is a test')") + + # Read from a readonly database + {:ok, ro} = Exqlite.open(path, [:readonly]) + on_exit(fn -> Exqlite.close(ro) end) + {:ok, stmt} = Exqlite.prepare(ro, "select rowid, stuff from test") + on_exit(fn -> Exqlite.finalize(stmt) end) + + assert {:row, [1, "This is a test"]} = Exqlite.step(ro, stmt) + + # Readonly database cannot insert + assert {:error, + %Exqlite.Error{code: 8, message: "attempt to write a readonly database"}} = + Exqlite.execute(ro, "insert into test (stuff) values ('This is a test')") + end + + @tag :tmp_dir + test "connects to a file with an accented character", %{tmp_dir: tmp_dir} do + path = Path.join(tmp_dir, "databasΓ©.sqlite") + assert {:ok, db} = Exqlite.open(path, [:create, :readwrite]) + on_exit(fn -> Exqlite.close(db) end) + assert is_reference(db) + end + + test "fails to open a database with invalid flag" do + assert_raise ArgumentError, ~r/invalid flag/, fn -> + Exqlite.open(":memory:", [:notarealflag]) + end + end + end + + describe ".close/1" do + test "raises on invalid db handle" do + db = make_ref() + assert_raise ErlangError, ~r/invalid connection/, fn -> Exqlite.close(db) end + end + + test "closes a database in memory" do + {:ok, db} = Exqlite.open(":memory:", [:readwrite]) + assert :ok = Exqlite.close(db) + end + + test "closing a database multiple times works" do + {:ok, db} = Exqlite.open(":memory:", [:readwrite]) + assert :ok = Exqlite.close(db) + assert :ok = Exqlite.close(db) + end + end + + describe ".execute/2" do + setup do + {:ok, db} = Exqlite.open(":memory:", [:readwrite]) + on_exit(fn -> Exqlite.close(db) end) + {:ok, db: db} + end + + test "creates a table", %{db: db} do + assert :ok = Exqlite.execute(db, "create table test(stuff text)") + assert :ok = Exqlite.execute(db, "insert into test(stuff) values('test')") + assert {:ok, 1} = Exqlite.last_insert_rowid(db) + assert {:ok, 1} = Exqlite.changes(db) + assert {:ok, [[1, "test"]]} = all(db, "select rowid, stuff from test") + end + + test "handles incorrect syntax", %{db: db} do + assert {:error, %Exqlite.Error{code: 1, message: "near \"a\": syntax error"}} = + Exqlite.execute(db, "create a dumb table test(stuff text)") + + assert {:ok, 0} = Exqlite.changes(db) + end + + @tag :skip + test "creates a virtual table with fts3", %{db: db} do + assert :ok = + Exqlite.execute( + db, + "create virtual table things using fts3(content text)" + ) + + assert :ok = + Exqlite.execute( + db, + "insert into things(content) values('this is content')" + ) + end + + @tag :skip + test "creates a virtual table with fts4", %{db: db} do + assert :ok = + Exqlite.execute( + db, + "create virtual table things using fts4(content text)" + ) + + assert :ok = + Exqlite.execute( + db, + "insert into things(content) values('this is content')" + ) + end + + @tag :skip + test "creates a virtual table with fts5", %{db: db} do + assert :ok = + Exqlite.execute(db, "create virtual table things using fts5(content)") + + assert :ok = + Exqlite.execute( + db, + "insert into things(content) values('this is content')" + ) + end + + test "handles unicode characters", %{db: db} do + assert :ok = Exqlite.execute(db, "create table test (stuff text)") + assert :ok = Exqlite.execute(db, "insert into test (stuff) values ('😝')") + assert {:ok, [[1, "😝"]]} = all(db, "select rowid, stuff from test") + end + end + + describe ".prepare/2" do + setup do + {:ok, db} = Exqlite.open(":memory:", [:readwrite]) + on_exit(fn -> Exqlite.close(db) end) + {:ok, db: db} + end + + test "preparing a valid sql statement", %{db: db} do + assert {:ok, stmt} = Exqlite.prepare(db, "select 1, 'hello'") + on_exit(fn -> Exqlite.finalize(stmt) end) + + assert {:row, [1, "hello"]} = Exqlite.step(db, stmt) + assert :done = Exqlite.step(db, stmt) + end + + test "supports utf8 in error messages", %{db: db} do + assert {:error, %Exqlite.Error{code: 1, message: "no such table: 🌍"} = error} = + Exqlite.prepare(db, "select * from 🌍") + + assert Exception.message(error) == "no such table: 🌍" + end + end + + describe ".finalize/1" do + setup do + {:ok, db} = Exqlite.open(":memory:", [:readwrite]) + on_exit(fn -> Exqlite.close(db) end) + {:ok, db: db} + end + + test "releases a statement", %{db: db} do + {:ok, stmt} = Exqlite.prepare(db, "select 1") + + assert :ok = Exqlite.finalize(stmt) + + # TODO improve SQLITE_MISUSE error message + # https://www.sqlite.org/rescode.html#misuse + assert {:error, %Exqlite.Error{code: 21, message: "not an error"}} = + Exqlite.step(db, stmt) + end + + test "double releasing a statement", %{db: db} do + {:ok, stmt} = Exqlite.prepare(db, "select 1") + assert :ok = Exqlite.finalize(stmt) + assert :ok = Exqlite.finalize(stmt) + end + end + + describe ".bind_all/3" do + setup do + {:ok, db} = Exqlite.open(":memory:", [:readwrite]) + on_exit(fn -> Exqlite.close(db) end) + {:ok, db: db} + end + + test "binding values to a valid sql statement", %{db: db} do + values = [1, "testing"] + + {:ok, stmt} = Exqlite.prepare(db, "select ?, ?") + on_exit(fn -> Exqlite.finalize(stmt) end) + + assert :ok = Exqlite.bind_all(db, stmt, values) + assert {:row, ^values} = Exqlite.step(db, stmt) + end + + test "trying to bind with incorrect amount of arguments", %{db: db} do + {:ok, stmt} = Exqlite.prepare(db, "select ?") + on_exit(fn -> Exqlite.finalize(stmt) end) + + assert_raise ErlangError, ~r/arguments wrong length/, fn -> + Exqlite.bind_all(db, stmt, []) + end + end + end + + describe ".columns/2" do + setup do + {:ok, db} = Exqlite.open(":memory:", [:readwrite]) + on_exit(fn -> Exqlite.close(db) end) + {:ok, db: db} + end + + test "returns the column definitions", %{db: db} do + {:ok, stmt} = Exqlite.prepare(db, "select 1 as id, 'hello' as stuff") + on_exit(fn -> Exqlite.finalize(stmt) end) + assert {:ok, ["id", "stuff"]} = Exqlite.columns(db, stmt) + end + + test "supports utf8 column names", %{db: db} do + {:ok, stmt} = Exqlite.prepare(db, "select 1 as πŸ‘‹, 'hello' as ✍️") + on_exit(fn -> Exqlite.finalize(stmt) end) + assert {:ok, ["πŸ‘‹", "✍️"]} = Exqlite.columns(db, stmt) + end + end + + describe ".step/2" do + setup do + {:ok, db} = Exqlite.open(":memory:", [:readwrite]) + on_exit(fn -> Exqlite.close(db) end) + {:ok, db: db} + end + + test "returns results", %{db: db} do + {:ok, stmt} = Exqlite.prepare(db, "select 1, 'test'") + on_exit(fn -> Exqlite.finalize(stmt) end) + + assert {:row, [1, "test"]} = Exqlite.step(db, stmt) + assert :done = Exqlite.step(db, stmt) + + assert {:row, [1, "test"]} = Exqlite.step(db, stmt) + assert :done = Exqlite.step(db, stmt) + end + + test "returns no results", %{db: db} do + {:ok, stmt} = Exqlite.prepare(db, "select * from sqlite_master") + on_exit(fn -> Exqlite.finalize(stmt) end) + + assert :done = Exqlite.step(db, stmt) + end + + test "works with insert", %{db: db} do + :ok = Exqlite.execute(db, "create table test(stuff text)") + + {:ok, stmt} = Exqlite.prepare(db, "insert into test(stuff) values(?1)") + on_exit(fn -> Exqlite.finalize(stmt) end) + + :ok = Exqlite.bind_all(db, stmt, ["this is a test"]) + assert :done = Exqlite.step(db, stmt) + + assert {:ok, [[1, "this is a test"]]} = all(db, "select rowid, stuff from test") + end + end + + describe ".multi_step/3" do + setup do + {:ok, db} = Exqlite.open(":memory:", [:readwrite]) + on_exit(fn -> Exqlite.close(db) end) + + :ok = Exqlite.execute(db, "create table test(stuff text)") + + rows = [ + ["one"], + ["two"], + ["three"], + ["four"], + ["five"], + ["six"] + ] + + {:ok, insert} = Exqlite.prepare(db, "insert into test(stuff) values(?1)") + :ok = Exqlite.insert_all(db, insert, rows) + :ok = Exqlite.finalize(insert) + + {:ok, db: db} + end + + test "returns results", %{db: db} do + {:ok, stmt} = Exqlite.prepare(db, "select rowid, stuff from test order by rowid") + on_exit(fn -> Exqlite.finalize(stmt) end) + + assert {:rows, rows} = Exqlite.multi_step(db, stmt, _steps = 4) + assert rows == [[1, "one"], [2, "two"], [3, "three"], [4, "four"]] + + assert {:done, rows} = Exqlite.multi_step(db, stmt, _steps = 4) + assert rows == [[5, "five"], [6, "six"]] + end + end + + # TODO move under .prepare + describe "working with prepared statements after close" do + test "returns proper error" do + {:ok, db} = Exqlite.open(":memory:", [:readwrite]) + + {:ok, stmt} = Exqlite.prepare(db, "select ?1") + on_exit(fn -> Exqlite.finalize(stmt) end) + + :ok = Exqlite.close(db) + :ok = Exqlite.bind_all(db, stmt, ["this is a test"]) + + # TODO improve SQLITE_MISUSE error message + assert {:error, %Exqlite.Error{code: 21, message: "out of memory"}} = + Exqlite.execute(db, "select 1") + + assert {:row, ["this is a test"]} = Exqlite.step(db, stmt) + assert :done = Exqlite.step(db, stmt) + end + end + + describe "serialize and deserialize" do + @tag :tmp_dir + test "serialize a database to binary and deserialize to new database", %{ + tmp_dir: tmp_dir + } do + path = Path.join(tmp_dir, "db.sqlite") + if File.exists?(path), do: File.rm!(path) + {:ok, db} = Exqlite.open(path, [:create, :readwrite]) + + :ok = Exqlite.execute(db, "create table test(stuff text)") + + assert {:ok, binary} = Exqlite.serialize(db, "main") + assert is_binary(binary) + :ok = Exqlite.close(db) + + {:ok, db} = Exqlite.open(":memory:", [:readwrite]) + assert :ok = Exqlite.deserialize(db, "main", binary) + + :ok = Exqlite.execute(db, "insert into test(stuff) values('hello')") + + {:ok, stmt} = Exqlite.prepare(db, "select rowid, stuff from test") + on_exit(fn -> Exqlite.finalize(stmt) end) + + assert {:row, [1, "hello"]} = Exqlite.step(db, stmt) + end + end + + describe "set_update_hook/2" do + defmodule ChangeListener do + use GenServer + + def start_link({parent, name}), + do: GenServer.start_link(__MODULE__, {parent, name}) + + def init({parent, name}), do: {:ok, {parent, name}} + + def handle_info({_action, _db, _table, _row_id} = change, {parent, name}) do + send(parent, {change, name}) + {:noreply, {parent, name}} + end + end + + @describetag :tmp_dir + + setup %{tmp_dir: tmp_dir} do + path = Path.join(tmp_dir, "db.sqlite") + + {:ok, db} = Exqlite.open(path, [:create, :readwrite]) + on_exit(fn -> Exqlite.close(db) end) + + :ok = Exqlite.execute(db, "create table test(num integer)") + {:ok, db: db, path: path} + end + + test "can listen to data change notifications", context do + {:ok, listener_pid} = ChangeListener.start_link({self(), :listener}) + Exqlite.set_update_hook(context.db, listener_pid) + + :ok = Exqlite.execute(context.db, "insert into test(num) values (10)") + :ok = Exqlite.execute(context.db, "insert into test(num) values (11)") + :ok = Exqlite.execute(context.db, "update test set num = 1000") + :ok = Exqlite.execute(context.db, "delete from test where num = 1000") + + assert_receive {{:insert, "main", "test", 1}, _}, 1000 + assert_receive {{:insert, "main", "test", 2}, _}, 1000 + assert_receive {{:update, "main", "test", 1}, _}, 1000 + assert_receive {{:update, "main", "test", 2}, _}, 1000 + assert_receive {{:delete, "main", "test", 1}, _}, 1000 + assert_receive {{:delete, "main", "test", 2}, _}, 1000 + end + + test "only one pid can listen at a time", context do + {:ok, listener1_pid} = ChangeListener.start_link({self(), :listener1}) + {:ok, listener2_pid} = ChangeListener.start_link({self(), :listener2}) + + Exqlite.set_update_hook(context.db, listener1_pid) + :ok = Exqlite.execute(context.db, "insert into test(num) values (10)") + assert_receive {{:insert, "main", "test", 1}, :listener1}, 1000 + + Exqlite.set_update_hook(context.db, listener2_pid) + :ok = Exqlite.execute(context.db, "insert into test(num) values (10)") + assert_receive {{:insert, "main", "test", 2}, :listener2}, 1000 + refute_receive {{:insert, "main", "test", 2}, :listener1}, 1000 + end + + test "notifications don't cross dbs", context do + {:ok, listener_pid} = ChangeListener.start_link({self(), :listener}) + {:ok, new_db} = Exqlite.open(context.path, [:readwrite]) + Exqlite.set_update_hook(new_db, listener_pid) + :ok = Exqlite.execute(context.db, "insert into test(num) values (10)") + refute_receive {{:insert, "main", "test", 1}, _}, 1000 + end + end + + describe "set_log_hook/1" do + setup do + {:ok, db} = Exqlite.open(":memory:", [:readwrite]) + on_exit(fn -> Exqlite.close(db) end) + {:ok, db: db} + end + + test "can receive errors", %{db: db} do + assert :ok = Exqlite.set_log_hook(self()) + + assert {:error, %Exqlite.Error{code: 1, message: "near \"some\": syntax error"}} = + Exqlite.prepare(db, "some invalid sql") + + assert_receive {:log, rc, msg} + assert rc == 1 + assert msg == "near \"some\": syntax error in \"some invalid sql\"" + refute_receive _anything_else + end + + test "only one pid can listen at a time", %{db: db} do + assert :ok = Exqlite.set_log_hook(self()) + + task = + Task.async(fn -> + :ok = Exqlite.set_log_hook(self()) + + assert {:error, + %Exqlite.Error{code: 1, message: "near \"some\": syntax error"}} = + Exqlite.prepare(db, "some invalid sql") + + assert_receive {:log, rc, msg} + assert rc == 1 + assert msg == "near \"some\": syntax error in \"some invalid sql\"" + refute_receive _anything_else + end) + + Task.await(task) + refute_receive _anything_else + end + + test "receives notifications from all dbs", %{db: db1} do + assert :ok = Exqlite.set_log_hook(self()) + + assert {:ok, db2} = Exqlite.open(":memory:", [:readwrite]) + on_exit(fn -> Exqlite.close(db2) end) + + assert {:error, _reason} = Exqlite.prepare(db1, "some invalid sql 1") + assert_receive {:log, rc, msg} + assert rc == 1 + assert msg == "near \"some\": syntax error in \"some invalid sql 1\"" + refute_receive _anything_else + + assert {:error, _reason} = Exqlite.prepare(db2, "some invalid sql 2") + assert_receive {:log, rc, msg} + assert rc == 1 + assert msg == "near \"some\": syntax error in \"some invalid sql 2\"" + refute_receive _anything_else + end + end + + describe ".interrupt/1" do + setup do + {:ok, db} = Exqlite.open(":memory:", [:readwrite]) + on_exit(fn -> Exqlite.close(db) end) + {:ok, db: db} + end + + test "double interrupting a db", %{db: db} do + assert :ok = Exqlite.interrupt(db) + assert :ok = Exqlite.interrupt(db) + end + + test "interrupting a long running query and able to close a db", %{db: db} do + test = self() + + spawn(fn -> + assert {:error, %Exqlite.Error{code: 9, message: "interrupted"}} = + all(db, """ + WITH RECURSIVE r(i) AS ( + VALUES(0) UNION ALL SELECT i FROM r LIMIT 1000000000 + ) + SELECT i FROM r WHERE i = 1 + """) + + send(test, :done) + end) + + Process.sleep(100) + assert :ok = Exqlite.interrupt(db) + assert {:ok, [[1]]} = all(db, "select 1") + + assert_receive :done + end + end +end diff --git a/test/exqlite/extensions_test.exs b/test/exqlite/extensions_test.exs index 3291bb67..f499b564 100644 --- a/test/exqlite/extensions_test.exs +++ b/test/exqlite/extensions_test.exs @@ -1,56 +1,59 @@ defmodule Exqlite.ExtensionsTest do - use ExUnit.Case + use ExUnit.Case, async: true - alias Exqlite.Basic + defp all(db, sql, args \\ []) do + {:ok, stmt} = Exqlite.prepare(db, sql) + on_exit(fn -> Exqlite.finalize(stmt) end) + + unless args == [] do + :ok = Exqlite.bind_all(db, stmt, args) + end + + Exqlite.fetch_all(db, stmt, 100) + end describe "enable_load_extension" do - test "loading can be enabled / disabled" do - {:ok, path} = Temp.path() - {:ok, conn} = Basic.open(path) - :ok = Basic.enable_load_extension(conn) + setup do + {:ok, db} = Exqlite.open(":memory:", [:readwrite]) + on_exit(fn -> :ok = Exqlite.close(db) end) + {:ok, db: db} + end - {:ok, [[nil]], _} = - Basic.load_extension(conn, ExSqlean.path_for("re")) |> Basic.rows() + test "loading can be enabled / disabled", %{db: db} do + assert :ok = Exqlite.enable_load_extension(db, true) - {:ok, [[1]], _} = - Basic.exec(conn, "select regexp_like('the year is 2021', '2021')") - |> Basic.rows() + re = ExSqlean.path_for("re") - :ok = Basic.disable_load_extension(conn) + assert {:ok, [[nil]]} = all(db, "select load_extension(?)", [re]) - {:error, "not authorized"} = - Basic.load_extension(conn, ExSqlean.path_for("re")) |> Basic.rows() - end + assert :ok = Exqlite.enable_load_extension(db, false) - test "works for 're' (regex)" do - {:ok, path} = Temp.path() - {:ok, conn} = Basic.open(path) + assert {:error, %Exqlite.Error{code: 1, message: "not authorized"}} = + all(db, "select load_extension(?)", [re]) + end - :ok = Basic.enable_load_extension(conn) + test "works for 're' (regex)", %{db: db} do + :ok = Exqlite.enable_load_extension(db, true) - {:ok, [[nil]], _} = - Basic.load_extension(conn, ExSqlean.path_for("re")) |> Basic.rows() + re = ExSqlean.path_for("re") + {:ok, _} = all(db, "select load_extension(?)", [re]) - {:ok, [[0]], _} = - Basic.exec(conn, "select regexp_like('the year is 2021', '2k21')") - |> Basic.rows() + assert {:ok, [[0]]} = + all(db, "select regexp_like('the year is 2021', ?)", ["2k21"]) - {:ok, [[1]], _} = - Basic.exec(conn, "select regexp_like('the year is 2021', '2021')") - |> Basic.rows() + assert {:ok, [[1]]} = + all(db, "select regexp_like('the year is 2021', ?)", ["2021"]) end - test "stats extension" do - {:ok, path} = Temp.path() - {:ok, conn} = Basic.open(path) + test "stats extension", %{db: db} do + :ok = Exqlite.enable_load_extension(db, true) - :ok = Basic.enable_load_extension(conn) - Basic.load_extension(conn, ExSqlean.path_for("stats")) - Basic.load_extension(conn, ExSqlean.path_for("series")) + for ext <- ["stats", "series"] do + {:ok, _} = all(db, "select load_extension(?)", [ExSqlean.path_for(ext)]) + end - {:ok, [[50.5]], ["median(value)"]} = - Basic.exec(conn, "select median(value) from generate_series(1, 100)") - |> Basic.rows() + assert {:ok, [[50.5]]} = + all(db, "select median(value) from generate_series(1, 100)") end end end diff --git a/test/exqlite/integration_test.exs b/test/exqlite/integration_test.exs index e860cc90..7fc1f3a7 100644 --- a/test/exqlite/integration_test.exs +++ b/test/exqlite/integration_test.exs @@ -1,201 +1,126 @@ defmodule Exqlite.IntegrationTest do - use ExUnit.Case + use ExUnit.Case, async: true - alias Exqlite.Connection - alias Exqlite.Sqlite3 - alias Exqlite.Query + describe "simple" do + setup do + {:ok, db} = Exqlite.open(":memory:", [:readwrite]) + on_exit(fn -> :ok = Exqlite.close(db) end) - test "simple prepare execute and close" do - path = Temp.path!() - {:ok, db} = Sqlite3.open(path) - :ok = Sqlite3.execute(db, "create table test (id ingeger primary key, stuff text)") - :ok = Sqlite3.close(db) + :ok = + Exqlite.execute(db, "create table test (id ingeger primary key, stuff text)") - {:ok, conn} = Connection.connect(database: path) + {:ok, db: db} + end - {:ok, query, _} = - %Exqlite.Query{statement: "SELECT * FROM test WHERE id = :id"} - |> Connection.handle_prepare([2], conn) + test "prepare, bind, execute, and step", %{db: db} do + assert :ok = + Exqlite.execute(db, "insert into test (id, stuff) values (1, 'hello')") - {:ok, _query, result, conn} = Connection.handle_execute(query, [2], [], conn) - assert result + assert {:ok, stmt} = + Exqlite.prepare(db, "select * from test where id = ?") - {:ok, _, conn} = Connection.handle_close(query, [], conn) - assert conn + on_exit(fn -> Exqlite.finalize(stmt) end) - File.rm(path) - end - - test "transaction handling with concurrent connections" do - path = Temp.path!() - - {:ok, conn1} = - Connection.connect( - database: path, - journal_mode: :wal, - cache_size: -64_000, - temp_store: :memory - ) - - {:ok, conn2} = - Connection.connect( - database: path, - journal_mode: :wal, - cache_size: -64_000, - temp_store: :memory - ) - - {:ok, _result, conn1} = Connection.handle_begin([], conn1) - assert conn1.transaction_status == :transaction - query = %Query{statement: "create table foo(id integer, val integer)"} - {:ok, _query, _result, conn1} = Connection.handle_execute(query, [], [], conn1) - {:ok, _result, conn1} = Connection.handle_rollback([], conn1) - assert conn1.transaction_status == :idle - - {:ok, _result, conn2} = Connection.handle_begin([], conn2) - assert conn2.transaction_status == :transaction - query = %Query{statement: "create table foo(id integer, val integer)"} - {:ok, _query, _result, conn2} = Connection.handle_execute(query, [], [], conn2) - {:ok, _result, conn2} = Connection.handle_rollback([], conn2) - assert conn2.transaction_status == :idle - - File.rm(path) - end - - test "handles busy correctly" do - path = Temp.path!() - - {:ok, conn1} = - Connection.connect( - database: path, - journal_mode: :wal, - cache_size: -64_000, - temp_store: :memory, - busy_timeout: 0 - ) - - {:ok, conn2} = - Connection.connect( - database: path, - journal_mode: :wal, - cache_size: -64_000, - temp_store: :memory, - busy_timeout: 0 - ) - - {:ok, _result, conn1} = Connection.handle_begin([mode: :immediate], conn1) - assert conn1.transaction_status == :transaction - {:disconnect, _err, conn2} = Connection.handle_begin([mode: :immediate], conn2) - assert conn2.transaction_status == :idle - {:ok, _result, conn1} = Connection.handle_commit([mode: :immediate], conn1) - assert conn1.transaction_status == :idle - {:ok, _result, conn2} = Connection.handle_begin([mode: :immediate], conn2) - assert conn2.transaction_status == :transaction - {:ok, _result, conn2} = Connection.handle_commit([mode: :immediate], conn2) - assert conn2.transaction_status == :idle - - Connection.disconnect(nil, conn1) - Connection.disconnect(nil, conn2) - - File.rm(path) - end - - test "transaction with interleaved connections" do - path = Temp.path!() - - {:ok, conn1} = - Connection.connect( - database: path, - journal_mode: :wal, - cache_size: -64_000, - temp_store: :memory - ) - - {:ok, conn2} = - Connection.connect( - database: path, - journal_mode: :wal, - cache_size: -64_000, - temp_store: :memory - ) - - {:ok, _result, conn1} = Connection.handle_begin([mode: :immediate], conn1) - query = %Query{statement: "create table foo(id integer, val integer)"} - {:ok, _query, _result, conn1} = Connection.handle_execute(query, [], [], conn1) - - # transaction overlap - {:ok, _result, conn2} = Connection.handle_begin([], conn2) - assert conn2.transaction_status == :transaction - {:ok, _result, conn1} = Connection.handle_rollback([], conn1) - assert conn1.transaction_status == :idle - - query = %Query{statement: "create table foo(id integer, val integer)"} - {:ok, _query, _result, conn2} = Connection.handle_execute(query, [], [], conn2) - {:ok, _result, conn2} = Connection.handle_rollback([], conn2) - assert conn2.transaction_status == :idle - - Connection.disconnect(nil, conn1) - Connection.disconnect(nil, conn2) - - File.rm(path) - end - - test "transaction handling with single connection" do - path = Temp.path!() + assert :ok = Exqlite.bind_all(db, stmt, [1]) - {:ok, conn1} = - Connection.connect( - database: path, - journal_mode: :wal, - cache_size: -64_000, - temp_store: :memory - ) + assert {:row, [1, "hello"]} = Exqlite.step(db, stmt) + assert :done = Exqlite.step(db, stmt) + end - {:ok, _result, conn1} = Connection.handle_begin([], conn1) - assert conn1.transaction_status == :transaction + test "insert_all and fetch_all", %{db: db} do + {:ok, insert} = Exqlite.prepare(db, "insert into test (id, stuff) values (?, ?)") + on_exit(fn -> Exqlite.finalize(insert) end) - query = %Query{statement: "create table foo(id integer, val integer)"} - {:ok, _query, _result, conn1} = Connection.handle_execute(query, [], [], conn1) - {:ok, _result, conn1} = Connection.handle_rollback([], conn1) - assert conn1.transaction_status == :idle + assert :ok = Exqlite.insert_all(db, insert, [[1, "hello"], [2, "world"]]) - {:ok, _result, conn1} = Connection.handle_begin([], conn1) - assert conn1.transaction_status == :transaction + {:ok, select} = Exqlite.prepare(db, "select * from test limit ?") + on_exit(fn -> Exqlite.finalize(select) end) - query = %Query{statement: "create table foo(id integer, val integer)"} - {:ok, _query, _result, conn1} = Connection.handle_execute(query, [], [], conn1) - {:ok, _result, conn1} = Connection.handle_rollback([], conn1) - assert conn1.transaction_status == :idle + assert :ok = Exqlite.bind_all(db, select, [100]) - File.rm(path) + assert {:ok, [[1, "hello"], [2, "world"]]} = + Exqlite.fetch_all(db, select, _steps = 100) + end end - test "exceeding timeout" do - path = Temp.path!() - - {:ok, conn} = - DBConnection.start_link(Connection, - idle_interval: 5_000, - database: path, - journal_mode: :wal, - cache_size: -64_000, - temp_store: :memory - ) - - query = %Query{statement: "create table foo(id integer, val integer)"} - {:ok, _, _} = DBConnection.execute(conn, query, []) - - values = for i <- 1..10_001, do: "(#{i}, #{i})" - - query = %Query{ - statement: "insert into foo(id, val) values #{Enum.join(values, ",")}" - } - - {:ok, _, _} = DBConnection.execute(conn, query, []) - - query = %Query{statement: "select * from foo"} - {:ok, _, _} = DBConnection.execute(conn, query, [], timeout: 1) - - File.rm(path) + describe "locks" do + @describetag :tmp_dir + + setup %{tmp_dir: tmp_dir} do + path = Path.join(tmp_dir, "db.sqlite") + if File.exists?(path), do: File.rm!(path) + + {:ok, db1} = Exqlite.open(path, [:create, :readwrite]) + on_exit(fn -> Exqlite.close(db1) end) + + {:ok, db2} = Exqlite.open(path, [:readwrite]) + on_exit(fn -> Exqlite.close(db2) end) + + {:ok, dbs: [db1, db2]} + end + + test "consecutive transactions", %{dbs: [db1, db2]} do + assert :ok = Exqlite.execute(db1, "begin") + assert :transaction = Exqlite.transaction_status(db1) + assert :ok = Exqlite.execute(db1, "create table foo(id integer, val integer)") + assert :ok = Exqlite.execute(db1, "rollback") + assert :idle = Exqlite.transaction_status(db1) + + assert :ok = Exqlite.execute(db2, "begin") + assert :transaction = Exqlite.transaction_status(db2) + assert :ok = Exqlite.execute(db2, "create table foo(id integer, val integer)") + assert :ok = Exqlite.execute(db2, "rollback") + assert :idle = Exqlite.transaction_status(db2) + end + + test "write lock", %{dbs: [db1, db2]} do + :ok = Exqlite.execute(db2, "pragma busy_timeout=0") + + assert :ok = Exqlite.execute(db1, "begin immediate") + assert :transaction = Exqlite.transaction_status(db1) + + # https://www.sqlite.org/rescode.html#busy + assert {:error, %Exqlite.Error{code: 5, message: "database is locked"}} = + Exqlite.execute(db2, "begin immediate") + + assert :idle = Exqlite.transaction_status(db2) + + assert :ok = Exqlite.execute(db1, "commit") + assert :idle = Exqlite.transaction_status(db1) + + assert :ok = Exqlite.execute(db2, "begin immediate") + assert :transaction = Exqlite.transaction_status(db2) + assert :ok = Exqlite.execute(db2, "commit") + assert :idle = Exqlite.transaction_status(db2) + end + + test "overlapped immediate/deferred transactions", %{dbs: [db1, db2]} do + assert :ok = Exqlite.execute(db1, "begin immediate") + assert :ok = Exqlite.execute(db1, "create table foo(id integer, val integer)") + + # transaction overlap + assert :ok = Exqlite.execute(db2, "begin") + assert :transaction = Exqlite.transaction_status(db2) + + assert :ok = Exqlite.execute(db1, "rollback") + assert :idle = Exqlite.transaction_status(db1) + + assert :ok = Exqlite.execute(db2, "create table foo(id integer, val integer)") + assert :ok = Exqlite.execute(db2, "rollback") + assert :idle = Exqlite.transaction_status(db2) + end + + test "transaction handling with single db", %{dbs: [db1, _db2]} do + assert :ok = Exqlite.execute(db1, "begin") + assert :transaction = Exqlite.transaction_status(db1) + assert :ok = Exqlite.execute(db1, "create table foo(id integer, val integer)") + assert :ok = Exqlite.execute(db1, "rollback") + assert :idle = Exqlite.transaction_status(db1) + assert :ok = Exqlite.execute(db1, "begin") + assert :transaction = Exqlite.transaction_status(db1) + assert :ok = Exqlite.execute(db1, "create table foo(id integer, val integer)") + assert :ok = Exqlite.execute(db1, "rollback") + assert :idle = Exqlite.transaction_status(db1) + end end end diff --git a/test/exqlite/pragma_test.exs b/test/exqlite/pragma_test.exs index 4da014c5..0088c810 100644 --- a/test/exqlite/pragma_test.exs +++ b/test/exqlite/pragma_test.exs @@ -1,123 +1,64 @@ defmodule Exqlite.PragmaTest do - use ExUnit.Case + use ExUnit.Case, async: true - alias Exqlite.Pragma + @moduletag :tmp_dir - test ".journal_mode/1" do - assert Pragma.journal_mode(journal_mode: :truncate) == "truncate" - assert Pragma.journal_mode(journal_mode: :persist) == "persist" - assert Pragma.journal_mode(journal_mode: :memory) == "memory" - assert Pragma.journal_mode(journal_mode: :wal) == "wal" - assert Pragma.journal_mode(journal_mode: :off) == "off" - assert Pragma.journal_mode(journal_mode: :delete) == "delete" - assert Pragma.journal_mode([]) == "delete" - assert Pragma.journal_mode(nil) == "delete" + setup %{tmp_dir: tmp_dir} do + path = Path.join(tmp_dir, "db.sqlite") + if File.exists?(path), do: File.rm!(path) - assert_raise( - ArgumentError, - "invalid :journal_mode", - fn -> - Pragma.journal_mode(journal_mode: :invalid) - end - ) + {:ok, db} = Exqlite.open(path, [:create, :readwrite]) + on_exit(fn -> :ok = Exqlite.close(db) end) - assert_raise( - ArgumentError, - "invalid :journal_mode", - fn -> - Pragma.journal_mode(journal_mode: "WAL") - end - ) + {:ok, db: db} end - test ".temp_store/1" do - assert Pragma.temp_store(temp_store: :memory) == 2 - assert Pragma.temp_store(temp_store: :file) == 1 - assert Pragma.temp_store(temp_store: :default) == 0 - assert Pragma.temp_store([]) == 0 - assert Pragma.temp_store(nil) == 0 - - assert_raise( - ArgumentError, - "invalid :temp_store", - fn -> - Pragma.temp_store(temp_store: :invalid) - end - ) - - assert_raise( - ArgumentError, - fn -> - Pragma.temp_store(temp_store: 1) - end - ) + defp one(db, sql) do + {:ok, stmt} = Exqlite.prepare(db, sql) + on_exit(fn -> Exqlite.finalize(stmt) end) + {:ok, [row]} = Exqlite.fetch_all(db, stmt, 100) + row end - test ".synchronous/1" do - assert Pragma.synchronous(synchronous: :extra) == 3 - assert Pragma.synchronous(synchronous: :full) == 2 - assert Pragma.synchronous(synchronous: :normal) == 1 - assert Pragma.synchronous(synchronous: :off) == 0 - assert Pragma.synchronous([]) == 1 - assert Pragma.synchronous(nil) == 1 + test "journal_mode", %{db: db} do + assert [_default = "delete"] = one(db, "pragma journal_mode") - assert_raise( - ArgumentError, - "invalid :synchronous", - fn -> - Pragma.synchronous(synchronous: :invalid) - end - ) + for mode <- ["wal", "memory", "off", "delete", "truncate", "persist"] do + :ok = Exqlite.execute(db, "pragma journal_mode=#{mode}") + assert [^mode] = one(db, "pragma journal_mode") + end end - test ".foreign_keys/1" do - assert Pragma.foreign_keys(foreign_keys: :on) == 1 - assert Pragma.foreign_keys(foreign_keys: :off) == 0 - assert Pragma.foreign_keys([]) == 1 - assert Pragma.foreign_keys(nil) == 1 + test "temp_store", %{db: db} do + assert [_default = 0] = one(db, "pragma temp_store") - assert_raise( - ArgumentError, - "invalid :foreign_keys", - fn -> - Pragma.foreign_keys(foreign_keys: :invalid) - end - ) + for {name, code} <- [{"memory", 2}, {"default", 0}, {"file", 1}] do + :ok = Exqlite.execute(db, "pragma temp_store=#{name}") + assert [^code] = one(db, "pragma temp_store") + end end - test ".cache_size/1" do - assert Pragma.cache_size(cache_size: -64_000) == -64_000 - assert Pragma.cache_size([]) == -2_000 - assert Pragma.cache_size(nil) == -2_000 + test "synchronous", %{db: db} do + assert [_full = 2] = one(db, "pragma synchronous") + + for {name, code} <- [{"extra", 3}, {"off", 0}, {"full", 2}, {"normal", 1}] do + :ok = Exqlite.execute(db, "pragma synchronous=#{name}") + assert [^code] = one(db, "pragma synchronous") + end end - test ".cache_spill/1" do - assert Pragma.cache_spill(cache_spill: :on) == 1 - assert Pragma.cache_spill(cache_spill: :off) == 0 - assert Pragma.cache_spill([]) == 1 - assert Pragma.cache_spill(nil) == 1 + test "foreign_keys", %{db: db} do + assert [_off = 0] = one(db, "pragma foreign_keys") - assert_raise( - ArgumentError, - "invalid :cache_spill", - fn -> - Pragma.cache_spill(cache_spill: :invalid) - end - ) + for {name, code} <- [{"on", 1}, {"off", 0}] do + :ok = Exqlite.execute(db, "pragma foreign_keys=#{name}") + assert [^code] = one(db, "pragma foreign_keys") + end end - test ".case_sensitive_like/1" do - assert Pragma.case_sensitive_like(case_sensitive_like: :on) == 1 - assert Pragma.case_sensitive_like(case_sensitive_like: :off) == 0 - assert Pragma.case_sensitive_like([]) == 0 - assert Pragma.case_sensitive_like(nil) == 0 - - assert_raise( - ArgumentError, - "invalid :case_sensitive_like", - fn -> - Pragma.case_sensitive_like(case_sensitive_like: :invalid) - end - ) + test "cache_size", %{db: db} do + assert [_default = -2000] = one(db, "pragma cache_size") + :ok = Exqlite.execute(db, "pragma cache_size=-64000") + assert [-64000] = one(db, "pragma cache_size") end end diff --git a/test/exqlite/query_test.exs b/test/exqlite/query_test.exs deleted file mode 100644 index 34a06135..00000000 --- a/test/exqlite/query_test.exs +++ /dev/null @@ -1,41 +0,0 @@ -defmodule Exqlite.QueryTest do - use ExUnit.Case - - setup :create_conn! - - test "table reader integration", %{conn: conn} do - assert {:ok, _} = Exqlite.query(conn, "CREATE TABLE tab(x integer, y text);", []) - - assert {:ok, _} = - Exqlite.query( - conn, - "INSERT INTO tab(x, y) VALUES (1, 'a'), (2, 'b'), (3, 'c');", - [] - ) - - assert {:ok, res} = - Exqlite.query( - conn, - "SELECT * FROM tab;", - [] - ) - - assert res |> Table.to_rows() |> Enum.to_list() == [ - %{"x" => 1, "y" => "a"}, - %{"x" => 2, "y" => "b"}, - %{"x" => 3, "y" => "c"} - ] - - columns = Table.to_columns(res) - assert Enum.to_list(columns["x"]) == [1, 2, 3] - assert Enum.to_list(columns["y"]) == ["a", "b", "c"] - end - - defp create_conn!(_) do - opts = [database: "#{Temp.path!()}.db"] - - {:ok, pid} = start_supervised(Exqlite.child_spec(opts)) - - [conn: pid] - end -end diff --git a/test/exqlite/sqlite3_test.exs b/test/exqlite/sqlite3_test.exs deleted file mode 100644 index 4a72185f..00000000 --- a/test/exqlite/sqlite3_test.exs +++ /dev/null @@ -1,623 +0,0 @@ -defmodule Exqlite.Sqlite3Test do - use ExUnit.Case - - alias Exqlite.Sqlite3 - - describe ".open/1" do - test "opens a database in memory" do - {:ok, conn} = Sqlite3.open(":memory:") - - assert conn - end - - test "opens a database on disk" do - {:ok, path} = Temp.path() - {:ok, conn} = Sqlite3.open(path) - - assert conn - - File.rm(path) - end - - test "creates database path on disk when non-existent" do - {:ok, path} = Temp.mkdir() - {:ok, conn} = Sqlite3.open(path <> "/non_exist.db") - - assert conn - - File.rm(path) - end - - test "opens a database in readonly mode" do - # Create database with readwrite connection - {:ok, path} = Temp.path() - {:ok, rw_conn} = Sqlite3.open(path) - - create_table_query = "create table test (id integer primary key, stuff text)" - :ok = Sqlite3.execute(rw_conn, create_table_query) - - insert_value_query = "insert into test (stuff) values ('This is a test')" - :ok = Sqlite3.execute(rw_conn, insert_value_query) - - # Read from database with a readonly connection - {:ok, ro_conn} = Sqlite3.open(path, mode: :readonly) - - select_query = "select id, stuff from test order by id asc" - {:ok, statement} = Sqlite3.prepare(ro_conn, select_query) - {:row, columns} = Sqlite3.step(ro_conn, statement) - - assert [1, "This is a test"] == columns - - # Readonly connection cannot insert - assert {:error, "attempt to write a readonly database"} == - Sqlite3.execute(ro_conn, insert_value_query) - end - - test "opens a database in a list of mode" do - # Create database with readwrite connection - {:ok, path} = Temp.path() - {:ok, rw_conn} = Sqlite3.open(path) - - create_table_query = "create table test (id integer primary key, stuff text)" - :ok = Sqlite3.execute(rw_conn, create_table_query) - - insert_value_query = "insert into test (stuff) values ('This is a test')" - :ok = Sqlite3.execute(rw_conn, insert_value_query) - - # Read from database with a readonly connection - {:ok, ro_conn} = Sqlite3.open(path, mode: [:readonly, :nomutex]) - - select_query = "select id, stuff from test order by id asc" - {:ok, statement} = Sqlite3.prepare(ro_conn, select_query) - {:row, columns} = Sqlite3.step(ro_conn, statement) - - assert [1, "This is a test"] == columns - end - - test "opens a database with invalid mode" do - {:ok, path} = Temp.path() - - msg = - "expected mode to be `:readwrite`, `:readonly` or list of modes, but received :notarealmode" - - assert_raise ArgumentError, msg, fn -> - Sqlite3.open(path, mode: :notarealmode) - end - end - - test "opens a database with invalid single nomutex mode" do - {:ok, path} = Temp.path() - - msg = - "expected mode to be `:readwrite` or `:readonly`, can't use a single :nomutex mode" - - assert_raise ArgumentError, msg, fn -> - Sqlite3.open(path, mode: :nomutex) - end - end - - test "opens a database with invalid list of mode" do - {:ok, path} = Temp.path() - - msg = - "expected mode to be `:readwrite`, `:readonly` or `:nomutex`, but received :notarealmode" - - assert_raise ArgumentError, msg, fn -> - Sqlite3.open(path, mode: [:notarealmode]) - end - end - end - - describe ".close/2" do - test "closes a database in memory" do - {:ok, conn} = Sqlite3.open(":memory:") - :ok = Sqlite3.close(conn) - end - - test "closing a database multiple times works properly" do - {:ok, conn} = Sqlite3.open(":memory:") - :ok = Sqlite3.close(conn) - :ok = Sqlite3.close(conn) - end - end - - describe ".execute/2" do - test "creates a table" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") - - :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('This is a test')") - {:ok, 1} = Sqlite3.last_insert_rowid(conn) - {:ok, 1} = Sqlite3.changes(conn) - :ok = Sqlite3.close(conn) - end - - test "handles incorrect syntax" do - {:ok, conn} = Sqlite3.open(":memory:") - - {:error, ~s|near "a": syntax error|} = - Sqlite3.execute( - conn, - "create a dumb table test (id integer primary key, stuff text)" - ) - - {:ok, 0} = Sqlite3.changes(conn) - :ok = Sqlite3.close(conn) - end - - test "creates a virtual table with fts3" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create virtual table things using fts3(content text)") - - :ok = - Sqlite3.execute(conn, "insert into things(content) VALUES ('this is content')") - end - - test "creates a virtual table with fts4" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create virtual table things using fts4(content text)") - - :ok = - Sqlite3.execute(conn, "insert into things(content) VALUES ('this is content')") - end - - test "creates a virtual table with fts5" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = Sqlite3.execute(conn, "create virtual table things using fts5(content)") - - :ok = - Sqlite3.execute(conn, "insert into things(content) VALUES ('this is content')") - end - - test "handles unicode characters" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Exqlite.Sqlite3.execute( - conn, - "create table test (id integer primary key, stuff text)" - ) - - :ok = Exqlite.Sqlite3.execute(conn, "insert into test (stuff) values ('😝')") - end - end - - describe ".prepare/3" do - test "preparing a valid sql statement" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") - - {:ok, statement} = Sqlite3.prepare(conn, "insert into test (stuff) values (?1)") - - assert statement - end - - test "supports utf8 in error messages" do - {:ok, conn} = Sqlite3.open(":memory:") - assert {:error, "no such table: 🌍"} = Sqlite3.prepare(conn, "select * from 🌍") - end - end - - describe ".release/2" do - test "double releasing a statement" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") - - {:ok, statement} = Sqlite3.prepare(conn, "insert into test (stuff) values (?1)") - :ok = Sqlite3.release(conn, statement) - :ok = Sqlite3.release(conn, statement) - end - - test "releasing a statement" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") - - {:ok, statement} = Sqlite3.prepare(conn, "insert into test (stuff) values (?1)") - :ok = Sqlite3.release(conn, statement) - end - - test "releasing a nil statement" do - {:ok, conn} = Sqlite3.open(":memory:") - :ok = Sqlite3.release(conn, nil) - end - end - - describe ".bind/3" do - test "binding values to a valid sql statement" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") - - {:ok, statement} = Sqlite3.prepare(conn, "insert into test (stuff) values (?1)") - :ok = Sqlite3.bind(conn, statement, ["testing"]) - end - - test "trying to bind with incorrect amount of arguments" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") - - {:ok, statement} = Sqlite3.prepare(conn, "insert into test (stuff) values (?1)") - {:error, :arguments_wrong_length} = Sqlite3.bind(conn, statement, []) - end - - test "binds datetime value as string" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") - - {:ok, statement} = Sqlite3.prepare(conn, "insert into test (stuff) values (?1)") - :ok = Sqlite3.bind(conn, statement, [DateTime.utc_now()]) - end - - test "binds date value as string" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") - - {:ok, statement} = Sqlite3.prepare(conn, "insert into test (stuff) values (?1)") - :ok = Sqlite3.bind(conn, statement, [Date.utc_today()]) - end - - test "raises an error when binding non UTC datetimes" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") - - {:ok, statement} = Sqlite3.prepare(conn, "insert into test (stuff) values (?1)") - - msg = "#DateTime<2021-08-25 13:23:25+00:00 UTC Europe/Berlin> is not in UTC" - - assert_raise ArgumentError, msg, fn -> - {:ok, dt} = DateTime.from_naive(~N[2021-08-25 13:23:25], "Etc/UTC") - # Sneak in other timezone without a tz database - other_tz = struct(dt, time_zone: "Europe/Berlin") - - Sqlite3.bind(conn, statement, [other_tz]) - end - end - end - - describe ".columns/2" do - test "returns the column definitions" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") - - {:ok, statement} = Sqlite3.prepare(conn, "select id, stuff from test") - - {:ok, columns} = Sqlite3.columns(conn, statement) - - assert ["id", "stuff"] == columns - end - - test "supports utf8 column names" do - {:ok, conn} = Sqlite3.open(":memory:") - :ok = Sqlite3.execute(conn, "create table test(πŸ‘‹ text, ✍️ text)") - {:ok, statement} = Sqlite3.prepare(conn, "select * from test") - assert {:ok, ["πŸ‘‹", "✍️"]} = Sqlite3.columns(conn, statement) - end - end - - describe ".step/2" do - test "returns results" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") - - :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('This is a test')") - {:ok, 1} = Sqlite3.last_insert_rowid(conn) - :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('Another test')") - {:ok, 2} = Sqlite3.last_insert_rowid(conn) - - {:ok, statement} = - Sqlite3.prepare(conn, "select id, stuff from test order by id asc") - - {:row, columns} = Sqlite3.step(conn, statement) - assert [1, "This is a test"] == columns - {:row, columns} = Sqlite3.step(conn, statement) - assert [2, "Another test"] == columns - assert :done = Sqlite3.step(conn, statement) - - {:row, columns} = Sqlite3.step(conn, statement) - assert [1, "This is a test"] == columns - {:row, columns} = Sqlite3.step(conn, statement) - assert [2, "Another test"] == columns - assert :done = Sqlite3.step(conn, statement) - end - - test "returns no results" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") - - {:ok, statement} = Sqlite3.prepare(conn, "select id, stuff from test") - assert :done = Sqlite3.step(conn, statement) - end - - test "works with insert" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") - - {:ok, statement} = Sqlite3.prepare(conn, "insert into test (stuff) values (?1)") - :ok = Sqlite3.bind(conn, statement, ["this is a test"]) - assert :done == Sqlite3.step(conn, statement) - end - end - - describe ".multi_step/3" do - test "returns results" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") - - :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('one')") - :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('two')") - :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('three')") - :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('four')") - :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('five')") - :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('six')") - - {:ok, statement} = - Sqlite3.prepare(conn, "select id, stuff from test order by id asc") - - {:rows, rows} = Sqlite3.multi_step(conn, statement, 4) - assert rows == [[1, "one"], [2, "two"], [3, "three"], [4, "four"]] - - {:done, rows} = Sqlite3.multi_step(conn, statement, 4) - assert rows == [[5, "five"], [6, "six"]] - end - end - - describe ".multi_step/2" do - test "returns results" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") - - :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('one')") - :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('two')") - :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('three')") - :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('four')") - :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('five')") - :ok = Sqlite3.execute(conn, "insert into test (stuff) values ('six')") - - {:ok, statement} = - Sqlite3.prepare(conn, "select id, stuff from test order by id asc") - - {:done, rows} = Sqlite3.multi_step(conn, statement) - - assert rows == [ - [1, "one"], - [2, "two"], - [3, "three"], - [4, "four"], - [5, "five"], - [6, "six"] - ] - end - end - - describe "working with prepared statements after close" do - test "returns proper error" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = - Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") - - {:ok, statement} = Sqlite3.prepare(conn, "insert into test (stuff) values (?1)") - :ok = Sqlite3.close(conn) - :ok = Sqlite3.bind(conn, statement, ["this is a test"]) - - {:error, message} = - Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)") - - assert message == "Sqlite3 was invoked incorrectly." - - assert :done == Sqlite3.step(conn, statement) - end - end - - describe "serialize and deserialize" do - test "serialize a database to binary and deserialize to new database" do - {:ok, path} = Temp.path() - {:ok, conn} = Sqlite3.open(path) - - :ok = - Sqlite3.execute(conn, "create table test(id integer primary key, stuff text)") - - assert {:ok, binary} = Sqlite3.serialize(conn, "main") - assert is_binary(binary) - Sqlite3.close(conn) - File.rm(path) - - {:ok, conn} = Sqlite3.open(":memory:") - assert :ok = Sqlite3.deserialize(conn, "main", binary) - - assert :ok = - Sqlite3.execute(conn, "insert into test(id, stuff) values (1, 'hello')") - - assert {:ok, statement} = Sqlite3.prepare(conn, "select id, stuff from test") - assert {:row, [1, "hello"]} = Sqlite3.step(conn, statement) - end - end - - describe "set_update_hook/2" do - defmodule ChangeListener do - use GenServer - - def start_link({parent, name}), - do: GenServer.start_link(__MODULE__, {parent, name}) - - def init({parent, name}), do: {:ok, {parent, name}} - - def handle_info({_action, _db, _table, _row_id} = change, {parent, name}) do - send(parent, {change, name}) - {:noreply, {parent, name}} - end - end - - setup do - {:ok, path} = Temp.path() - {:ok, conn} = Sqlite3.open(path) - :ok = Sqlite3.execute(conn, "create table test(num integer)") - - on_exit(fn -> - Sqlite3.close(conn) - File.rm(path) - end) - - [conn: conn, path: path] - end - - test "can listen to data change notifications", context do - {:ok, listener_pid} = ChangeListener.start_link({self(), :listener}) - Sqlite3.set_update_hook(context.conn, listener_pid) - - :ok = Sqlite3.execute(context.conn, "insert into test(num) values (10)") - :ok = Sqlite3.execute(context.conn, "insert into test(num) values (11)") - :ok = Sqlite3.execute(context.conn, "update test set num = 1000") - :ok = Sqlite3.execute(context.conn, "delete from test where num = 1000") - - assert_receive {{:insert, "main", "test", 1}, _}, 1000 - assert_receive {{:insert, "main", "test", 2}, _}, 1000 - assert_receive {{:update, "main", "test", 1}, _}, 1000 - assert_receive {{:update, "main", "test", 2}, _}, 1000 - assert_receive {{:delete, "main", "test", 1}, _}, 1000 - assert_receive {{:delete, "main", "test", 2}, _}, 1000 - end - - test "only one pid can listen at a time", context do - {:ok, listener1_pid} = ChangeListener.start_link({self(), :listener1}) - {:ok, listener2_pid} = ChangeListener.start_link({self(), :listener2}) - - Sqlite3.set_update_hook(context.conn, listener1_pid) - :ok = Sqlite3.execute(context.conn, "insert into test(num) values (10)") - assert_receive {{:insert, "main", "test", 1}, :listener1}, 1000 - - Sqlite3.set_update_hook(context.conn, listener2_pid) - :ok = Sqlite3.execute(context.conn, "insert into test(num) values (10)") - assert_receive {{:insert, "main", "test", 2}, :listener2}, 1000 - refute_receive {{:insert, "main", "test", 2}, :listener1}, 1000 - end - - test "notifications don't cross connections", context do - {:ok, listener_pid} = ChangeListener.start_link({self(), :listener}) - {:ok, new_conn} = Sqlite3.open(context.path) - Sqlite3.set_update_hook(new_conn, listener_pid) - :ok = Sqlite3.execute(context.conn, "insert into test(num) values (10)") - refute_receive {{:insert, "main", "test", 1}, _}, 1000 - end - end - - describe "set_log_hook/1" do - setup do - {:ok, conn} = Sqlite3.open(":memory:") - on_exit(fn -> Sqlite3.close(conn) end) - {:ok, conn: conn} - end - - test "can receive errors", %{conn: conn} do - assert :ok = Sqlite3.set_log_hook(self()) - - assert {:error, reason} = Sqlite3.prepare(conn, "some invalid sql") - assert reason == "near \"some\": syntax error" - - assert_receive {:log, rc, msg} - assert rc == 1 - assert msg == "near \"some\": syntax error in \"some invalid sql\"" - refute_receive _anything_else - end - - test "only one pid can listen at a time", %{conn: conn} do - assert :ok = Sqlite3.set_log_hook(self()) - - task = - Task.async(fn -> - :ok = Sqlite3.set_log_hook(self()) - assert {:error, reason} = Sqlite3.prepare(conn, "some invalid sql") - assert reason == "near \"some\": syntax error" - assert_receive {:log, rc, msg} - assert rc == 1 - assert msg == "near \"some\": syntax error in \"some invalid sql\"" - refute_receive _anything_else - end) - - Task.await(task) - refute_receive _anything_else - end - - test "receives notifications from all connections", %{conn: conn1} do - assert :ok = Sqlite3.set_log_hook(self()) - assert {:ok, conn2} = Sqlite3.open(":memory:") - on_exit(fn -> Sqlite3.close(conn2) end) - - assert {:error, _reason} = Sqlite3.prepare(conn1, "some invalid sql 1") - assert_receive {:log, rc, msg} - assert rc == 1 - assert msg == "near \"some\": syntax error in \"some invalid sql 1\"" - refute_receive _anything_else - - assert {:error, _reason} = Sqlite3.prepare(conn2, "some invalid sql 2") - assert_receive {:log, rc, msg} - assert rc == 1 - assert msg == "near \"some\": syntax error in \"some invalid sql 2\"" - refute_receive _anything_else - end - end - - describe ".interrupt/1" do - test "double interrupting a connection" do - {:ok, conn} = Sqlite3.open(":memory:") - - :ok = Sqlite3.interrupt(conn) - :ok = Sqlite3.interrupt(conn) - end - - test "interrupting a nil connection" do - :ok = Sqlite3.interrupt(nil) - end - - test "interrupting a long running query and able to close a connection" do - {:ok, conn} = Sqlite3.open(":memory:") - - spawn(fn -> - :ok = - Sqlite3.execute( - conn, - "WITH RECURSIVE r(i) AS ( VALUES(0) UNION ALL SELECT i FROM r LIMIT 1000000000 ) SELECT i FROM r WHERE i = 1;" - ) - end) - - Process.sleep(100) - :ok = Sqlite3.interrupt(conn) - - :ok = Sqlite3.close(conn) - end - end -end diff --git a/test/exqlite/timeout_segfault_test.exs b/test/exqlite/timeout_segfault_test.exs deleted file mode 100644 index 35776d50..00000000 --- a/test/exqlite/timeout_segfault_test.exs +++ /dev/null @@ -1,41 +0,0 @@ -defmodule Exqlite.TimeoutSegfaultTest do - use ExUnit.Case - - @moduletag :slow_test - - setup do - {:ok, path} = Temp.path() - on_exit(fn -> File.rm(path) end) - - %{path: path} - end - - test "segfault", %{path: path} do - {:ok, conn} = - DBConnection.start_link(Exqlite.Connection, - busy_timeout: 50_000, - pool_size: 50, - timeout: 1, - database: path, - journal_mode: :wal - ) - - query = %Exqlite.Query{statement: "create table foo(id integer, val integer)"} - {:ok, _, _} = DBConnection.execute(conn, query, []) - - values = for i <- 1..1000, do: "(#{i}, #{i})" - statement = "insert into foo(id, val) values #{Enum.join(values, ",")}" - insert_query = %Exqlite.Query{statement: statement} - - 1..5000 - |> Task.async_stream(fn _ -> - try do - DBConnection.execute(conn, insert_query, [], timeout: 1) - catch - kind, reason -> - IO.puts("Error: #{inspect(kind)} reason: #{inspect(reason)}") - end - end) - |> Stream.run() - end -end diff --git a/test/test_helper.exs b/test/test_helper.exs index 6682d142..869559e7 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1 +1 @@ -ExUnit.start(capture_log: true, timeout: 120_000, exclude: [:slow_test]) +ExUnit.start()