Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 8 additions & 13 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
[![Coverage Status](https://coveralls.io/repos/github/fast-programmer/outboxer/badge.svg)](https://coveralls.io/github/fast-programmer/outboxer)
[![Join our Discord](https://img.shields.io/badge/Discord-blue?style=flat&logo=discord&logoColor=white)](https://discord.gg/x6EUehX6vU)

**Outboxer** is an implementation of the [transactional outbox pattern](https://docs.aws.amazon.com/prescriptive-guidance/latest/cloud-design-patterns/transactional-outbox.html) for event driven Ruby on Rails applications.
**Outboxer** is an implementation of the [transactional outbox pattern](https://docs.aws.amazon.com/prescriptive-guidance/latest/cloud-design-patterns/transactional-outbox.html) for **Ruby on Rails** applications.

It helps you migrate to **event-driven architecture** with at least once delivery guarantees.

# Quickstart

Expand All @@ -15,7 +17,7 @@ bundle add outboxer
bundle install
```

**2. Generate schema migrations and tests**
**2. Generate schema migrations, publisher script and tests**

```bash
bundle exec rails g outboxer:install
Expand All @@ -34,7 +36,7 @@ bundle exec rails generate model Event type:string created_at:datetime --skip-ti
bundle exec rails db:migrate
```

**5. Queue outboxer message when event created**
**5. Queue outboxer message after event created**

```ruby
# app/models/event.rb
Expand Down Expand Up @@ -70,26 +72,19 @@ Accountify::InvoiceRaisedEvent.create!(created_at: Time.current)

**8. Observe transactional consistency**

```shell
```log
TRANSACTION (0.2ms) BEGIN
Event Create (0.4ms) INSERT INTO "events" ...
Outboxer::Message Create (0.3ms) INSERT INTO "outboxer_messages" ...
TRANSACTION (0.2ms) COMMIT
```

**9. Publish outboxer message in block**
**8. Publish outboxer messages**

```ruby
# bin/outboxer_publisher

Outboxer::Publisher.publish_message do |publisher, message|
logger.info(
"Publishing outboxer message " \
"publisher_id=#{publisher[:id]} " \
"messageable_type=#{message[:messageable_type]} " \
"messageable_id=#{message[:messageable_id]}"
)

# TODO: publish message here
end
```
Expand All @@ -104,7 +99,7 @@ To ensure you have end to end coverage:

# Monitoring

Monitor multithreaded publishers using Outboxer's built-in web UI:
Monitor using the built-in web UI:

## Publishers

Expand Down
27 changes: 11 additions & 16 deletions lib/outboxer/message.rb
Original file line number Diff line number Diff line change
Expand Up @@ -127,8 +127,7 @@ def queue(messageable: nil, messageable_type: nil, messageable_id: nil,
# - `:messageable_type` [String]
# - `:messageable_id` [Integer, String]
# @yieldreturn [Object] result of the block (ignored)
# @return [Boolean] `true` if a message was published successfully,
# `false` if no queued message existed or publishing failed
# @return [Hash, nil] the same yielded message hash, or `nil` if no queued message exists
# @raise [Exception] re-raises non-StandardError exceptions after marking failed
# @example Publish with processing
# Outboxer::Message.publish(logger: logger) do |message|
Expand All @@ -137,22 +136,20 @@ def queue(messageable: nil, messageable_type: nil, messageable_id: nil,
def publish(hostname: Socket.gethostname,
process_id: Process.pid,
thread_id: Thread.current.object_id,
logger: nil, time: ::Time, &block)
raise ArgumentError, "publish requires a block" if block.nil?

logger: nil, time: ::Time)
publishing_started_at = time.now.utc

message = Message.publishing(
hostname: hostname, process_id: process_id, thread_id: thread_id, time: time)

if message.nil?
false
else
logger&.info(
"Outboxer message publishing id=#{message[:id]} " \
"messageable_type=#{message[:messageable_type]} " \
"messageable_id=#{message[:messageable_id]}")
return if message.nil?

logger&.info(
"Outboxer message publishing id=#{message[:id]} " \
"messageable_type=#{message[:messageable_type]} " \
"messageable_id=#{message[:messageable_id]}")

if block_given?
begin
yield message
rescue StandardError => error
Expand All @@ -169,8 +166,6 @@ def publish(hostname: Socket.gethostname,
"duration_ms=#{((time.now.utc - publishing_started_at) * 1000).to_i}\n" \
"#{error.class}: #{error.message}\n" \
"#{error.backtrace.join("\n")}")

false
rescue Exception => error
publishing_failed(
id: message[:id], lock_version: message[:lock_version], error: error,
Expand All @@ -196,10 +191,10 @@ def publish(hostname: Socket.gethostname,
"messageable_type=#{message[:messageable_type]} " \
"messageable_id=#{message[:messageable_id]} " \
"duration_ms=#{((time.now.utc - publishing_started_at) * 1000).to_i}")

true
end
end

message
end

# Selects and locks the next available message in **queued** status
Expand Down
215 changes: 139 additions & 76 deletions lib/outboxer/models/thread.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,20 +7,130 @@ class Thread < ActiveRecord::Base
HISTORIC_PROCESS_ID = 0
HISTORIC_THREAD_ID = 0

STATUS_COLUMNS = {
queued: ["queued_message_count", "queued_message_count_last_updated_at"],
publishing: ["publishing_message_count", "publishing_message_count_last_updated_at"],
published: ["published_message_count", "published_message_count_last_updated_at"],
failed: ["failed_message_count", "failed_message_count_last_updated_at"]
}.freeze
POSTGRES_SQL = <<~SQL.freeze
INSERT INTO outboxer_threads (
hostname,
process_id,
thread_id,
queued_message_count,
queued_message_count_last_updated_at,
publishing_message_count,
publishing_message_count_last_updated_at,
published_message_count,
published_message_count_last_updated_at,
failed_message_count,
failed_message_count_last_updated_at,
created_at,
updated_at
)
VALUES (
$1, $2, $3,
$4, $5,
$6, $7,
$8, $9,
$10, $11,
$12, $13
)
ON CONFLICT (hostname, process_id, thread_id)
DO UPDATE SET
queued_message_count =
outboxer_threads.queued_message_count +
EXCLUDED.queued_message_count,
queued_message_count_last_updated_at =
CASE
WHEN EXCLUDED.queued_message_count != 0
THEN EXCLUDED.queued_message_count_last_updated_at
ELSE outboxer_threads.queued_message_count_last_updated_at
END,
publishing_message_count =
outboxer_threads.publishing_message_count +
EXCLUDED.publishing_message_count,
publishing_message_count_last_updated_at =
CASE
WHEN EXCLUDED.publishing_message_count != 0
THEN EXCLUDED.publishing_message_count_last_updated_at
ELSE outboxer_threads.publishing_message_count_last_updated_at
END,
published_message_count =
outboxer_threads.published_message_count +
EXCLUDED.published_message_count,
published_message_count_last_updated_at =
CASE
WHEN EXCLUDED.published_message_count != 0
THEN EXCLUDED.published_message_count_last_updated_at
ELSE outboxer_threads.published_message_count_last_updated_at
END,
failed_message_count =
outboxer_threads.failed_message_count +
EXCLUDED.failed_message_count,
failed_message_count_last_updated_at =
CASE
WHEN EXCLUDED.failed_message_count != 0
THEN EXCLUDED.failed_message_count_last_updated_at
ELSE outboxer_threads.failed_message_count_last_updated_at
END,
updated_at = EXCLUDED.updated_at
SQL

MYSQL_SQL = <<~SQL.freeze
INSERT INTO outboxer_threads (
hostname,
process_id,
thread_id,
queued_message_count,
queued_message_count_last_updated_at,
publishing_message_count,
publishing_message_count_last_updated_at,
published_message_count,
published_message_count_last_updated_at,
failed_message_count,
failed_message_count_last_updated_at,
created_at,
updated_at
)
VALUES (
?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
ON DUPLICATE KEY UPDATE
queued_message_count =
queued_message_count + VALUES(queued_message_count),
queued_message_count_last_updated_at =
IF(
VALUES(queued_message_count) != 0,
VALUES(queued_message_count_last_updated_at),
queued_message_count_last_updated_at
),
publishing_message_count =
publishing_message_count +
VALUES(publishing_message_count),
publishing_message_count_last_updated_at =
IF(
VALUES(publishing_message_count) != 0,
VALUES(publishing_message_count_last_updated_at),
publishing_message_count_last_updated_at
),
published_message_count =
published_message_count +
VALUES(published_message_count),
published_message_count_last_updated_at =
IF(
VALUES(published_message_count) != 0,
VALUES(published_message_count_last_updated_at),
published_message_count_last_updated_at
),
failed_message_count =
failed_message_count + VALUES(failed_message_count),
failed_message_count_last_updated_at =
IF(
VALUES(failed_message_count) != 0,
VALUES(failed_message_count_last_updated_at),
failed_message_count_last_updated_at
),
updated_at = VALUES(updated_at)
SQL

BASE_INSERT_COLUMNS = %w[
hostname
process_id
thread_id
created_at
updated_at
].freeze
SQL_QUERY_NAME = "Outboxer::Models::Thread.update_message_counts_by!".freeze

def self.update_message_counts_by!(
hostname: Socket.gethostname,
Expand All @@ -32,70 +142,23 @@ def self.update_message_counts_by!(
failed_message_count: 0,
current_utc_time: Time.now.utc
)
is_postgres = connection.adapter_name.downcase.include?("postgres")

insert_columns = BASE_INSERT_COLUMNS.dup
insert_values = [hostname, process_id, thread_id, current_utc_time, current_utc_time]

update_columns = []
update_values = []

{
queued: queued_message_count,
publishing: publishing_message_count,
published: published_message_count,
failed: failed_message_count
}.each do |name, message_count|
next if message_count.to_i == 0
@is_postgres ||= connection.adapter_name.downcase.include?("postgres")

message_count_column, last_updated_column = STATUS_COLUMNS.fetch(name)

insert_columns << message_count_column
insert_columns << last_updated_column
insert_values << message_count
insert_values << current_utc_time

if is_postgres
update_columns <<
"#{message_count_column} = " \
"#{table_name}.#{message_count_column} + " \
"EXCLUDED.#{message_count_column}"
else
update_columns << "#{message_count_column} = #{message_count_column} + ?"
update_values << message_count
end

update_columns << "#{last_updated_column} = ?"
update_values << current_utc_time
end

update_columns << "updated_at = ?"
update_values << current_utc_time

insert_sql = <<~SQL
INSERT INTO #{table_name} (#{insert_columns.join(", ")})
VALUES (#{(["?"] * insert_columns.length).join(", ")})
SQL

sql =
if is_postgres
<<~SQL
#{insert_sql}
ON CONFLICT (hostname, process_id, thread_id)
DO UPDATE SET
#{update_columns.join(",\n ")}
SQL
else
<<~SQL
#{insert_sql}
ON DUPLICATE KEY UPDATE
#{update_columns.join(",\n ")}
SQL
end

connection.exec_query(
sanitize_sql_array([sql, *insert_values, *update_values])
)
connection.exec_update(@is_postgres ? POSTGRES_SQL : MYSQL_SQL, SQL_QUERY_NAME, [
hostname,
process_id,
thread_id,
queued_message_count,
current_utc_time,
publishing_message_count,
current_utc_time,
published_message_count,
current_utc_time,
failed_message_count,
current_utc_time,
current_utc_time,
current_utc_time
])
end
end
end
Expand Down
4 changes: 2 additions & 2 deletions lib/outboxer/publisher.rb
Original file line number Diff line number Diff line change
Expand Up @@ -576,7 +576,7 @@ def create_publisher_thread(id:, index:,
while !terminating?
if publishing?
begin
message_published = Message.publish(logger: logger) do |message|
published_message = Message.publish(logger: logger) do |message|
block.call({ id: id }, message)
end
rescue StandardError => error
Expand All @@ -590,7 +590,7 @@ def create_publisher_thread(id:, index:,
process: process,
kernel: kernel)
else
if message_published == false
if published_message.nil?
Publisher.sleep(
poll_interval,
tick_interval: tick_interval,
Expand Down
Loading
Loading