Merge pull request #6 from ora-io/dev

Dev

Author: murongg

README.md

@@ -1,12 +1,12 @@
 # ORA STACK
 
 ## What is ORA STACK
-ORA Stack is a comprehensive framework designed for quickly setting up a web3 Oracle Service.
+ORA Stack is a comprehensive framework designed for quickly setting up a robust web3 Oracle Service.
 
 It includes:
-- [Orap](./packages/orap/): the declarative signal-based backend framework;
-- [Reku](./packages/reku/): the reliable ethereum toolkit and utils;
-- [Utils](./packages/utils/): the common swiss-knife package.
+- [Orap](./packages/orap/): A Declarative Robust Oracle Backend Framework
+- [Reku](./packages/reku/): A Reliable Ethereum Kit & Utils
+- [Utils](./packages/utils/): A Common Swiss-knife Utils Package
 
 Comming soon:
 - UAO: the async oracle framework

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ora-stack",
-  "version": "0.0.8",
+  "version": "0.1.0",
   "private": true,
   "packageManager": "[email protected]",
   "description": "",

packages/orap/README.md

@@ -23,14 +23,191 @@
 
 ORAP is a declarative framework for building oracle services, handy to use out of the box.
 
+## Owl The Rapper
+> Show me you `Flow`s, I'll help you `assemble` to `Verse`s, which compose into a `Orap`. 
+>
+> `drop` the `Beat`s, let's `play`!
+
+`Orap` provides 2 styles of usage:
+- OO Style (Basic):
+  - Use this framework as a basic tool set.
+  - example: [customDemo](./examples/customDemo)
+  - it's more flexible but cumbersome somehow. 
+    - e.g. you can use your own storage other than Redis and Memory, e.g. mysql etc., for caching.
+    - you can define your own Task structure and handle workflow.
+- Declarative Style (Rap-lized): 
+  - Use this as a declarative *Rap-lized* framework, writing oracle services just like rapping! **Coding Like a Rapper**
+  - example: [declarativeDemo](./examples/declarativeDemo)
+  - it's way more easy to implement, `Orap` handles most of the common part, e.g. signal handle, task defining, task caching, task fetch and processing, multitasks processing, etc., while it may sacrifice flexibility in some way.
+
+Back in the scene, there are 2 internal layers in `Orap`:
+
+- Basic Layer: 
+  - mainly referring to the `Signal`, `StoreManager`, and `Task`, where the concepts are self-explained in engineering context.
+  - it can be used directly by users.
+- *Rap-lized* Layer: 
+  - mainly referring to the `Flow`, `Verse`, and `Beat`, where the concepts are introduced by `Orap` only. 
+  - it helps to build the declarative functionality, which is way easier for users and save some developers.
+  - it mostly for internal developing purpose, and ~~should be~~ easy to scale and extend, though user also has access to them if they want.
+
+
+> About Multi-chain: Currently Each `Orap` listens to only 1 blockchain network by design, similar to http servers. Create multiple `Orap` instances to implement multi-chain listener.
+> 
+> Suggest to include network in task `prefix()`, to avoid key collision in cache store
+
+
 ## Usage
+
+### Declarative Style (Rap-lized)
+
+It comes with rich features like customized task cache, multitasks handling etc.
+
+Note the following already includes using Redis as the store to cache tasks, allowing continuation after service restart, it's robust even when the service restarts.
+
 ```ts
-import { ListenOptions, Orap, StoreManager } from '../../orap'
-import { memoryStore, redisStore } from '../../utils'
+import { ethers } from 'ethers'
+import { Orap, StoreManager } from '@orap-io/orap'
+import { Logger, redisStore } from '@ora-io/utils'
+
+// new orap
+const orap = new Orap()
+
+// use redis
+const store = redisStore()
+const sm = new StoreManager(store)
+
+// use a logger
+const logger = new Logger('info', '[orap-raplize-sample]')
+
+const handle1 = (...args: any) => { logger.log('handle task 1', args); return true }
+
+const handle2 = (...args: any) => { logger.log('handle task 2', args); return true }
+
+// define event signal with crosscheck, and customized cacheable tasks
+// note: use redis as the cache layer
+orap.event(eventSignalParam)
+  .crosscheck(ccOptions)
+  // add a task
+  .task()
+  .cache(sm)
+  .prefix('ora-stack:orap:raplizeSample:Task-1:', 'ora-stack:orap:raplizeSample:Done-Task-1:')
+  .ttl({ taskTtl: 120000, doneTtl: 60000 })
+  .handle(handle1)
+  // add another task
+  .another()
+  .task()
+  .cache(sm)
+  .prefix('ora-stack:orap:raplizeSample:Task-2:', 'ora-stack:orap:raplizeSample:Done-Task-2:')
+  .ttl({ taskTtl: 120000, doneTtl: 60000 })
+  .handle(handle2)
+
+// set logger before listen
+orap.logger(logger)
+
+// start signal listeners
+orap.listen(
+  {
+    wsProvider: new ethers.WebSocketProvider('wss://127.0.0.1'),
+    httpProvider: new ethers.JsonRpcProvider('http://127.0.0.1')
+  },
+  () => { console.log('listening on provider.network') }
+)
+```
+
+#### Orap Flow
+
+Each `new Orap()` starts a `Orap Flow`
+
+**.event(eventSignalParam, handlFn)**
+- `eventSignalParam`: defines an event signal and enters an Event Flow
+- `handlFn`: customized hook on new event received. 
+  - `return true` to continue the rest of processes
+  - `return false` to hijack the rest of processes
+
+**.listen(options, onListenFn?)**
+- `options`:
+  - required: wsProvider, for subscription
+  - optional: httpProvider, for crosscheck only, since crosscheck is based on getLogs
+- `onListenFn`: customized hook when listener started.
+
+**.logger(logger)**
+- set which logger to use across this orap
+
+#### Event Flow
+
+Each `.event(...)` starts an `Event Flow`
+
+**.crosscheck(...)**
+- set an automated crosschecker for this event, to ensure the missing events of subscription will always be caught by `getLogs`. 
+- this can mitigate the common unstable nature of WebSocket rpc providers and increase the service availability.
+
+**.task()**
+- add a task for this event type
+- starts a `Task Flow`
+
+**.handle(handlFn)**
+- same as `.event(.., handlFn)`
+- `handlFn`: customized hook on new event received. 
+  - `return true` to continue the rest of processes
+  - `return false` to hijack the rest of processes
+
+**.another()**
+- back to the parent `Orap Flow`, so that it can add another `.event`
+- e.g. `orap.event(...).another().event(...)`
+
+#### Task Flow
+
+Each `.task(...)` starts a `Task Flow`
+
+**.handle(handler: HandleFn)**
+- set the task handler, the most important property for a task.
+  - `return true` to identify handle success, and entering `onSuccess`
+  - `return false` to identify handle failed, and entering `onSuccess`
+
+**.cache(sm: StoreManager)**
+- set the store to cache the tasks
+- default: use memory as the cache layer
+
+**.prefix(taskPrefix: Prefix, donePrefix: Prefix)**
+- set the prefix of tasks in the store cache for management
+- `donePrefix`: prefix of 'todo' tasks records
+- `donePrefix`: prefix of 'done' tasks records
+- default: "Task:" & "Done-Task:"
+
+**.ttl({ taskTtl, doneTtl }: { taskTtl: number; doneTtl: number })**
+- set the ttl of tasks in the store cache for management
+- `donePrefix`: ttl of 'todo' tasks records
+- `donePrefix`: ttl of 'done' tasks records
+- default: no limit
+
+**.key(toKey: ToKeyFn)**
+- defines the primary key of a task based on the event values (i.e. log topics)
+- default: random hex string
+
+**.success(onSuccess: HandleResultFn)**
+- defines how to process the task if the handler success
+- default: remove the 'todo' task from & set the 'done' task record to the cache store
+
+**.fail(onFail: HandleResultFn)**
+- defines how to process the task if the handler success
+- default: remove the 'todo' task from (ignore task)
+
+**.context(ctx: Context)**
+- optional: set the context that can be accessed to task functions
+
+**.another()**
+- back to the parent `Event Flow`, so that it can add another `.task`
+- e.g. `orap.event(...).task().another().task()`
+
+### OO Style (Basic)
+
+Note the following doesn't include task cache, it only calls `handle` every time it receives an event. So this service is only for demo, don't use it for production, otherwise it may miss events when service down.
+
+```ts
+import { ethers } from 'ethers'
+import { Orap } from '@orap-io/orap'
 
 const orap = new Orap()
-// const sm = new StoreManager(redisStore(...)) // use redis
-const sm = new StoreManager() // use memory
 
 const eventSignalParam = {
   address: '0xdAC17F958D2ee523a2206206994597C13D831ec7',
@@ -44,37 +221,85 @@ orap.event(eventSignalParam, handle)
   .crosscheck({ pollingInterval: 1000, batchBlocksCount: 1, blockInterval: 12000 })
 
 orap.listen(
-  { wsProvider: 'wss://127.0.0.1', httpProvider: 'http://127.0.0.1' },
+  {
+    wsProvider: new ethers.WebSocketProvider('wss://127.0.0.1'),
+    httpProvider: new ethers.JsonRpcProvider('http://127.0.0.1')
+  },
   () => { console.log('listening on provider.network') }
 )
 ```
 
-### listen options
-- required: wsProvider, for subscription
-- optional: httpProvider, for crosscheck only, since crosscheck is based on getLogs
+## *Rap-lized* Layer
 
-## Task
+The following terminology is internally, can be transparent to users.
 
-### TaskBase
-- provide universal `toString`, `fromString`, `stringify` 
+- A `Orap` compromises multiple `Verses` as the processors;
+- Some `Verses` includes `Beat`s, which define the pace and the incoming signals that triggering task handling in Orap.
+- For users want to build a `Orap`: only need to define `Flow`s **intuitively**, the Owl Rapper will take care of all the rest things.
 
-### TaskStorable
-- provide store (redis) compatible features, i.e. load, save, remove, done
-- overwrite when extends:
-  - `toKey()` (required): define the primary key that identifies each task, **doesn't** include `taskPrefix`
-  - `taskPrefix` (recommend): set the prefix of all tasks, also is used when `load` task
-  - `taskPrefixDone` (recommend): set the prefix of finished tasks, only used in `done`; no need to set if you don't use "task.done(sm)"
+**Terminology**
 
-## Signal
+- `Flow`: 
+  - handling user-defined option flows, 
+  - e.g. user can define following flows:
+    ```typescript
+    new Orap().event(..).crosscheck()
+    .handle(..)
+    .task(..).key(..).prefix(..).ttl(..)
+    .handle(..)
+    .another()
+    .task(..).key(..).prefix(..).ttl(..)
+    .handle(..)
+    ```
+- `Flow.assemble()`: 
+  - wrap up the `Flow` definition and build a `Verse` based on it.
+- `Verse`: 
+  - equivalent to an executor/processor of the corresponding `Flow`.
+- `Verse.play()`: 
+  - equivalent to start/launch the executor/processor.
+- `Beat`: 
+  - a wrap of the `Signal`, which defines the incoming triggers that initiate the runtime process flow
+    - e.g. `EventBeat` defines the event listener
+  - `Beat` wraps `Signal` into a uniformed class with only the `constructor` and `drop()`, easy for `Verse` to handle
+  - **Beats Drives the Song!**
+- `Beat.drop()`:
+  - start the `Signal` listener process. 
+  - **Drop the Beats!**
 
-all actions that arrive the oracle server and trigger actions are defined as signal, including:
-- [x] event
-- [ ] block
+## Basic Layer
+
+Basic Layer currently consists of 3 parts: 
+- `Signal` defines the incoming trigger types
+- `Task` defines the task types that handles signals
+- `StorageManager` defines the cache interface, allowing tasks to be cached
+
+### Signal
+
+All events that arrive the oracle service and trigger following actions are defined as `Signal`, including:
+- [x] `EventSignal`
+- [ ] `BlockSignal`
 - [ ] http request
 etc.
 
-### EventSignal
+**EventSignal**
 - define event listener as simple as: `orap.event({address:"0x", abi:"", eventName: "Transfer"}, handleSignal)`
-- provide crosschecker by `reku`, available config please checkout `AutoCrossCheckParam` in `reku`
-- currently one and only one crosschecker is set to each event signal
-- store: provide 2 options: use memory or redis, checkout `orap/store`
+- natively integrate `crosschecker` features from [@ora-io/reku](https://github.com/ora-io/ora-stack/blob/main/packages/reku/), available config please check out `AutoCrossCheckParam` in `reku`
+- each event signal only accept at most one crosschecker.
+- `callback`: the user provided handle function to handle the new signals.
+
+### Task
+
+**TaskBase**
+- provide universal `toString`, `fromString`, `stringify` 
+
+**TaskStorable**
+- provide store compatible features, i.e. load, save, remove, done
+- overwrite when extends:
+  - `toKey()` (required): define the primary key that identifies each task, **doesn't** include `taskPrefix`
+  - `taskPrefix` (recommend): set the prefix of all tasks, also is used when `load` task
+  - `taskPrefixDone` (recommend): set the prefix of finished tasks, only used in `done`; no need to set if you don't use "task.done(sm)"
+
+### StorageManager
+- a wrap class designed for caching tasks in Orap
+- `store`: the store entity, currently provides 2 options: use memory or redis, checkout `orap/store`
+- `queryDelay`: when doing retry-able operations, e.g. get all keys with the given prefix, this defines the interval between retries.

packages/orap/beat/event.ts

@@ -0,0 +1,25 @@
+import type { Logger } from '@ora-io/utils'
+import type { AutoCrossCheckParam, Providers } from '@ora-io/reku'
+import type { EventSignalCallback, EventSignalRegisterParams } from '../signal'
+import { EventSignal } from '../signal'
+
+/**
+ * Beat is Rap-lized, i.e. formalized in Orap framework, version of Signal
+ * only Signals have corresponding Beat class, task & orap don't have
+ */
+export class EventBeat extends EventSignal {
+  constructor(
+    params: EventSignalRegisterParams,
+    callback: EventSignalCallback,
+    logger: Logger,
+    crosscheckOptions: Omit<AutoCrossCheckParam, 'address' | 'topics' | 'onMissingLog'> | undefined,
+    private subscribeProvider: Providers,
+    private crosscheckProvider: Providers | undefined,
+  ) {
+    super(params, callback, logger, crosscheckOptions)
+  }
+
+  drop() {
+    this.listen(this.subscribeProvider, this.crosscheckProvider)
+  }
+}

packages/orap/beat/index.ts

@@ -0,0 +1 @@
+export type { EventBeat } from './event'

packages/orap/mock/demo/app.ts renamed to packages/orap/examples/customDemo/app.ts

@@ -8,13 +8,12 @@ import { TransferTask } from './taskTransfer'
 
 // new orap
 const orap = new Orap()
-// set to app specific logger
-orap.setLogger(logger)
+orap.logger(logger)
 
 let store: any
 let sm: any
 
-export function startDemo(options: ListenOptions, storeConfig?: any) {
+export function startCustomDemo(options: ListenOptions, storeConfig?: any) {
   store = redisStore(storeConfig) // use redis
   // store = memoryStore(storeConfig); // use memory
   sm = new StoreManager(store)

packages/orap/mock/demo/config.ts renamed to packages/orap/examples/customDemo/config.ts

@@ -35,17 +35,17 @@ export class TransferTask extends TaskStorable {
 
   static async load<T extends TaskStorable>(this: Constructor<T>, sm: any): Promise<T> {
     const task = await (this as any)._load(sm)
-    logger.log('[*] load task', task.toKey())
+    logger.log('[*] load task', await task.toKey())
     return task
   }
 
   async done(sm: any) {
-    logger.log('[*] done task', this.toKey())
+    logger.log('[*] done task', await this.toKey())
     await super.done(sm)
   }
 
   async remove(sm: any) {
-    logger.log('[*] remove task', this.toKey())
+    logger.log('[*] remove task', await this.toKey())
     await super.remove(sm)
   }
 }