rm dbconnection

Author: ruslandoga

.formatter.exs

@@ -1,7 +1,7 @@
 [
   inputs: [
     "{mix,.formatter}.exs",
-    "{config,lib,test}/**/*.{ex,exs}"
+    "{config,bench,lib,test}/**/*.{ex,exs}"
   ],
   line_length: 88
 ]

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## Unreleased
+
+- removed: `db_connection`. It's just a NIF now. `db_connection` is moved to `ecto_sqlite3`
+- removed: `Exqlite.Basic`
+
 ## v0.24.1
 
 - fixed: Pre-compile images for Apple and Windows

Makefile

@@ -94,18 +94,11 @@ CFLAGS += -DHAVE_USLEEP=1
 #       installing the nif. Just need to have certain environment variables
 #       enabled to support them.
 CFLAGS += -DALLOW_COVERING_INDEX_SCAN=1
-CFLAGS += -DENABLE_FTS3_PARENTHESIS=1
 CFLAGS += -DENABLE_LOAD_EXTENSION=1
-CFLAGS += -DENABLE_SOUNDEX=1
 CFLAGS += -DENABLE_STAT4=1
 CFLAGS += -DENABLE_UPDATE_DELETE_LIMIT=1
-CFLAGS += -DSQLITE_ENABLE_FTS3=1
-CFLAGS += -DSQLITE_ENABLE_FTS4=1
-CFLAGS += -DSQLITE_ENABLE_FTS5=1
-CFLAGS += -DSQLITE_ENABLE_GEOPOLY=1
 CFLAGS += -DSQLITE_ENABLE_MATH_FUNCTIONS=1
 CFLAGS += -DSQLITE_ENABLE_RBU=1
-CFLAGS += -DSQLITE_ENABLE_RTREE=1
 CFLAGS += -DSQLITE_OMIT_DEPRECATED=1
 CFLAGS += -DSQLITE_ENABLE_DBSTAT_VTAB=1
 
@@ -142,7 +135,11 @@ $(ARCHIVE_NAME): $(OBJ)
 $(PREFIX) $(BUILD):
 	mkdir -p $@
 
-clean:
+rebuild:
+	$(RM) $(LIB_NAME) $(BUILD)/sqlite3_nif.o
+	$(MAKE) all
+
+clean: 
 	$(RM) $(LIB_NAME) $(ARCHIVE_NAME) $(OBJ)
 
 .PHONY: all clean

README.md

@@ -15,20 +15,11 @@ Package: https://hex.pm/packages/exqlite
 
 ## Caveats
 
-* Prepared statements are not cached.
 * Prepared statements are not immutable. You must be careful when manipulating
   statements and binding values to statements. Do not try to manipulate the
   statements concurrently. Keep it isolated to one process.
-* Simultaneous writing is not supported by SQLite3 and will not be supported
-  here.
-* All native calls are run through the Dirty NIF scheduler.
-* Datetimes are stored without offsets. This is due to how SQLite3 handles date
-  and times. If you would like to store a timezone, you will need to create a
-  second column somewhere storing the timezone name and shifting it when you
-  get it from the database. This is more reliable than storing the offset as
-  `+03:00` as it does not respect daylight savings time.
-* When storing `BLOB` values, you have to use `{:blob, the_binary}`, otherwise
-  it will be interpreted as a string.
+* Some native calls are run through the Dirty NIF scheduler.
+  Some are executed directly on current scheduler.
 
 ## Installation
 
@@ -42,15 +33,6 @@ end
 
 
 ## Configuration
-
-### Runtime Configuration
-
-```elixir
-config :exqlite, default_chunk_size: 100
-```
-
-* `default_chunk_size` - The chunk size that is used when multi-stepping when
-  not specifying the chunk size explicitly.
 
 ### Compile-time Configuration
 
@@ -126,47 +108,52 @@ export EXQLITE_SYSTEM_CFLAGS=-I/usr/local/include/sqlcipher
 export EXQLITE_SYSTEM_LDFLAGS=-L/usr/local/lib -lsqlcipher
 ```
 
-Once you have `exqlite` configured, you can use the `:key` option in the database config to enable encryption:
+Once you have `exqlite` build configured, you can use the `key` pragma to enable encryption:
 
 ```elixir
-config :exqlite, key: "super-secret'
+{:ok, db} = Exqlite.open("sqlcipher.db")
+:ok = Exqlite.execute(db, "pragma key='super-secret'")
 ```
 
 ## Usage
 
-The `Exqlite.Sqlite3` module usage is fairly straight forward.
+The `Exqlite` module usage is fairly straight forward.
 
 ```elixir
-# We'll just keep it in memory right now
-{:ok, conn} = Exqlite.Sqlite3.open(":memory:")
+{:ok, db} = Exqlite.open("app.db", [:readwrite, :create])
+
+:ok = Exqlite.execute(db, "pragma foreign_keys=on")
+:ok = Exqlite.execute(db, "pragma journal_mode=wal")
+:ok = Exqlite.execute(db, "pragma busy_timeout=5000")
 
 # Create the table
-:ok = Exqlite.Sqlite3.execute(conn, "create table test (id integer primary key, stuff text)")
+:ok = Exqlite.execute(db, "create table test (id integer primary key, stuff text)")
 
 # Prepare a statement
-{:ok, statement} = Exqlite.Sqlite3.prepare(conn, "insert into test (stuff) values (?1)")
-:ok = Exqlite.Sqlite3.bind(conn, statement, ["Hello world"])
+{:ok, insert} = Exqlite.prepare(db, "insert into test (stuff) values (?1)")
+:ok = Exqlite.bind_all(db, insert, ["Hello world"])
 
 # Step is used to run statements
-:done = Exqlite.Sqlite3.step(conn, statement)
+:done = Exqlite.step(db, insert)
 
 # Prepare a select statement
-{:ok, statement} = Exqlite.Sqlite3.prepare(conn, "select id, stuff from test")
+{:ok, select} = Exqlite.prepare(db, "select id, stuff from test")
 
 # Get the results
-{:row, [1, "Hello world"]} = Exqlite.Sqlite3.step(conn, statement)
+{:row, [1, "Hello world"]} = Exqlite.step(db, select)
 
 # No more results
-:done = Exqlite.Sqlite3.step(conn, statement)
+:done = Exqlite.step(db, select)
 
-# Release the statement.
+# Release the statements.
 #
 # It is recommended you release the statement after using it to reclaim the memory
 # asap, instead of letting the garbage collector eventually releasing the statement.
 #
 # If you are operating at a high load issuing thousands of statements, it would be
 # possible to run out of memory or cause a lot of pressure on memory.
-:ok = Exqlite.Sqlite3.release(conn, statement)
+:ok = Exqlite.finalize(insert)
+:ok = Exqlite.finalize(select)
 ```
 
 ### Using SQLite3 native extensions
@@ -177,52 +164,39 @@ available by installing the [ExSqlean](https://github.com/mindreframer/ex_sqlean
 package. This package wraps [SQLean: all the missing SQLite functions](https://github.com/nalgeon/sqlean).
 
 ```elixir
-alias Exqlite.Basic
-{:ok, conn} = Basic.open("db.sqlite3")
-:ok = Basic.enable_load_extension(conn)
+{:ok, db} = Exqlite.open(":memory:", [:readwrite])
+:ok = Exqlite.enable_load_extension(db, true)
+
+exec = fn db, sql, params ->
+  with {:ok, stmt} <- Exqlite.prepare(db, sql) do
+    try do
+      with :ok <- Exqlite.bind_all(db, stmt, params) do
+        Exqlite.fetch_all(db, stmt)
+      end
+    after
+      Exqlite.finalize(stmt)
+    end
+  end
+end
 
 # load the regexp extension - https://github.com/nalgeon/sqlean/blob/main/docs/re.md
-Basic.load_extension(conn, ExSqlean.path_for("re"))
+{:ok, _rows} = exec.(db, "select load_extension(?)", [ExSqlean.path_for("re")])
 
 # run some queries to test the new `regexp_like` function
-{:ok, [[1]], ["value"]} = Basic.exec(conn, "select regexp_like('the year is 2021', ?) as value", ["2021"]) |> Basic.rows()
-{:ok, [[0]], ["value"]} = Basic.exec(conn, "select regexp_like('the year is 2021', ?) as value", ["2020"]) |> Basic.rows()
+{:ok, [[1]], ["value"]} = exec.(db, "select regexp_like('the year is 2021', ?) as value", ["2021"])
+{:ok, [[0]], ["value"]} = exec.(db, "select regexp_like('the year is 2021', ?) as value", ["2020"])
 
 # prevent loading further extensions
-:ok = Basic.disable_load_extension(conn)
-{:error, %Exqlite.Error{message: "not authorized"}, _} = Basic.load_extension(conn, ExSqlean.path_for("re"))
+:ok = Exqlite.enable_load_extension(db, false)
 
-# close connection
-Basic.close(conn)
-```
-
-It is also possible to load extensions using the `Connection` configuration. For example:
+{:error, %Exqlite.Error{message: "not authorized"}} =
+  exec.(db, "select load_extension(?)", [ExSqlean.path_for("stats")])
 
-```elixir
-arch_dir =
-  System.cmd("uname", ["-sm"])
-  |> elem(0)
-  |> String.trim()
-  |> String.replace(" ", "-")
-  |> String.downcase() # => "darwin-arm64"
-
-config :myapp, arch_dir: arch_dir
-
-# global
-config :exqlite, load_extensions: [ "./priv/sqlite/\#{arch_dir}/rotate" ]
-
-# per connection in a Phoenix app
-config :myapp, Myapp.Repo,
-  database: "path/to/db",
-  load_extensions: [
-    "./priv/sqlite/\#{arch_dir}/vector0",
-    "./priv/sqlite/\#{arch_dir}/vss0"
-  ]
+# close connection
+Exqlite.close(db)
 ```
 
-See [Exqlite.Connection.connect/1](https://hexdocs.pm/exqlite/Exqlite.Connection.html#connect/1)
-for more information. When using extensions for SQLite3, they must be compiled
-for the environment you are targeting.
+When using extensions for SQLite3, they must be compiled for the environment you are targeting.
 
 ## Why SQLite3
 
@@ -239,7 +213,7 @@ that would be resiliant to power outages and still maintain some state that
 
 ## Under The Hood
 
-We are using the Dirty NIF scheduler to execute the sqlite calls. The rationale
+We are using the Dirty NIF scheduler to execute most of the sqlite calls. The rationale
 behind this is that maintaining each sqlite's connection command pool is
 complicated and error prone.
 

bench/bind.exs

@@ -0,0 +1,7 @@
+{:ok, db} = Exqlite.open(":memory:", [:readwrite, :nomutex])
+{:ok, stmt} = Exqlite.prepare(db, "select ? + 1")
+
+Benchee.run(%{
+  "bind_all" => fn -> Exqlite.bind_all(db, stmt, [1]) end,
+  "dirty_cpu_bind_all" => fn -> Exqlite.dirty_cpu_bind_all(db, stmt, [1]) end
+})

bench/insert_all.exs

@@ -0,0 +1,18 @@
+{:ok, db} = Exqlite.open(":memory:", [:readwrite])
+:ok = Exqlite.execute(db, "create table test (id integer primary key, name text)")
+{:ok, stmt} = Exqlite.prepare(db, "insert into test(name) values(?)")
+
+Benchee.run(
+  %{
+    "insert_all" =>
+      {fn rows -> Exqlite.insert_all(db, stmt, rows) end,
+       before_scenario: fn _input -> Exqlite.execute(db, "truncate test") end}
+  },
+  inputs: %{
+    "3 rows" => Enum.map(1..3, fn i -> ["name-#{i}"] end),
+    "30 rows" => Enum.map(1..30, fn i -> ["name-#{i}"] end),
+    "90 rows" => Enum.map(1..90, fn i -> ["name-#{i}"] end),
+    "300 rows" => Enum.map(1..300, fn i -> ["name-#{i}"] end),
+    "1000 rows" => Enum.map(1..1000, fn i -> ["name-#{i}"] end)
+  }
+)

bench/prepare.exs

@@ -0,0 +1,58 @@
+tmp_dir = Path.expand(Path.join("./tmp", "bench/step"))
+File.mkdir_p!(tmp_dir)
+
+path = Path.join(tmp_dir, "db.sqlite")
+if File.exists?(path), do: File.rm!(path)
+
+IO.puts("Creating DB at #{path} ...")
+{:ok, db} = Exqlite.open(path, [:readwrite, :nomutex, :create])
+
+IO.puts("Inserting 1000 rows ...")
+:ok = Exqlite.execute(db, "create table test(stuff text)")
+{:ok, insert} = Exqlite.prepare(db, "insert into test(stuff) values(?)")
+:ok = Exqlite.insert_all(db, insert, Enum.map(1..1000, fn i -> ["name-#{i}"] end))
+:ok = Exqlite.finalize(insert)
+
+select = fn limit ->
+  {:ok, select} = Exqlite.prepare(db, "select * from test limit #{limit}")
+  select
+end
+
+defmodule Bench do
+  def step_all(db, stmt) do
+    case Exqlite.step(db, stmt) do
+      {:row, _} -> step_all(db, stmt)
+      :done -> :ok
+    end
+  end
+
+  def dirty_io_step_all(db, stmt) do
+    case Exqlite.dirty_io_step(db, stmt) do
+      {:row, _} -> dirty_io_step_all(db, stmt)
+      :done -> :ok
+    end
+  end
+
+  def multi_step_all(db, stmt, steps) do
+    case Exqlite.multi_step(db, stmt, steps) do
+      {:rows, _} -> multi_step_all(db, stmt, steps)
+      {:done, _} -> :ok
+    end
+  end
+end
+
+IO.puts("Running benchmarks ...\n")
+
+Benchee.run(
+  %{
+    "step" => fn stmt -> Bench.step_all(db, stmt) end,
+    "dirty_io_step" => fn stmt -> Bench.dirty_io_step_all(db, stmt) end,
+    "multi_step(100)" => fn stmt -> Bench.multi_step_all(db, stmt, _steps = 100) end
+  },
+  inputs: %{
+    "10 rows" => select.(10),
+    "100 rows" => select.(100),
+    "500 rows" => select.(500)
+  },
+  memory_time: 2
+)