Merge branch 'develop' into feat/cleanup

* develop: (42 commits)
  feat: rosetta data api and construction validation with rosetta-cli
  feat: stx addr encoding LRU cache
  feat: add nft_custody pg materialized view to speed up nft event lookup
  fix: build/publish Dockerfile python installation issues
  Add utils module with script to display top address balances
  chore: standardize updates between websockets and socket.io
  fix: transaction broadcast log
  chore: change @kyranjamie to @rafaelcr on pr template
  chore: rename github references from blockstack/ to hirosystems/
  chore: fix flaky BNS tests #723
  feat: endpoint for list of transactions #647
  chore: update c32check and cross-sha256 (including transitive) dependencies
  feat: production-capable CPU profiling #641
  chore: add read-only to readme
  fix: export api version variable for resolution
  fix: microblock related re-org bug causing txs to be incorrectly orphaned #804 #818
  fix: api versioning
  fix(docs): ensure naming convention is followed
  fix: rosetta account/balance speed
  feat: add broadcast/confirmed tx logs
  ...

Author: wileyj

.env

@@ -17,6 +17,14 @@ PG_SSL=false
 # Enable to have stacks-node events streamed to a file while the application is running
 # STACKS_EXPORT_EVENTS_FILE=/tmp/stacks-events.tsv
 
+# Initializes the API in read-only mode i.e. without an event server that listens to events from a node and writes them
+# to the local database. The API will only read data from the PG database specified above to respond to requests.
+# STACKS_READ_ONLY_MODE=true
+
+# If specified, an http server providing profiling capability endpoints will be opened on the given port.
+# This port should not be publicly exposed.
+# STACKS_PROFILER_PORT=9119
+
 STACKS_CORE_EVENT_PORT=3700
 STACKS_CORE_EVENT_HOST=127.0.0.1
 
@@ -49,6 +57,9 @@ BTC_FAUCET_PK=29c028009a8331358adcc61bb6397377c995d327ac0343ed8e8f1d4d3ef85c27
 TESTNET_SEND_MANY_CONTRACT_ID=STR8P3RD1EHA8AA37ERSSSZSWKS9T2GYQFGXNA4C.send-many-memo
 MAINNET_SEND_MANY_CONTRACT_ID=SP3FBR2AGK5H9QBDH3EEN6DF8EK8JY7RX8QJ5SVTE.send-many-memo
 
+# Enable debug logging
+# STACKS_API_LOG_LEVEL=debug
+
 # Directory containing Stacks 1.0 BNS data extracted from https://storage.googleapis.com/blockstack-v1-migration-data/export-data.tar.gz
 # BNS_IMPORT_DIR=/extracted/export-data-dir/
 
@@ -73,3 +84,6 @@ MAINNET_SEND_MANY_CONTRACT_ID=SP3FBR2AGK5H9QBDH3EEN6DF8EK8JY7RX8QJ5SVTE.send-man
 # IMGIX_DOMAIN=https://<your domain>.imgix.net
 # IMGIX_TOKEN=<your token>
 
+# Specify max number of STX address to store in an in-memory LRU cache (CPU optimization).
+# Defaults to 50,000, which should result in around 25 megabytes of additional memory usage.
+# STACKS_ADDRESS_CACHE_SIZE=10000

.eslintignore

@@ -15,3 +15,4 @@ src/tests-rosetta-cli/
 src/tests-bns/
 client/src/
 config/
+utils/src/

.eslintrc.js

@@ -9,7 +9,7 @@ module.exports = {
     ecmaVersion: 2020,
     sourceType: 'module',
   },
-  ignorePatterns: ['lib/*', 'client/*'],
+  ignorePatterns: ['lib/*', 'client/*', 'utils/*'],
   rules: {
     'prettier/prettier': 'error',
 

.github/ISSUE_TEMPLATE/testnet-bug.md

@@ -34,8 +34,6 @@ that demonstrates the issue.
 
 ----
 
-If you think this is eligible for a [bug bounty](https://testnet.blockstack.org/bounties), please check the relevant boxes below:
-
 ### Critical, Launch Blocking Bugs
 **Consensus critical bugs**
 - [ ] Can cause a chain split

.github/PULL_REQUEST_TEMPLATE.md

@@ -11,7 +11,7 @@ Describe the changes that where made in this pull request. When possible start w
 5. Provide examples of use cases with code samples and applicable acceptance criteria
 
 Example:
-As a Blockstack developer, I would like to encrypt files using the app private key. This is needed because storing unencrypted files is unacceptable. This pull request adds the `encryptContent` function which will take a string and encrypt it using the app private key.
+As a Hiro Systems developer, I would like to encrypt files using the app private key. This is needed because storing unencrypted files is unacceptable. This pull request adds the `encryptContent` function which will take a string and encrypt it using the app private key.
 
 ```
 encryptContent('my data')
@@ -41,7 +41,7 @@ Workarounds for or expected timeline for deprecation
     - Change in instructions inside tutorials/guides
     - etc...
 
-   The best way to find these is by searching inside the docs at https://github.com/blockstack/docs
+   The best way to find these is by searching inside the docs at https://github.com/hirosystems/docs
 -->
 - [ ] Link to documentation updates: 
 
@@ -60,4 +60,4 @@ Provide context on how tests should be performed.
 - [ ] Unit test coverage for new or modified code paths
 - [ ] `npm run test` passes
 - [ ] Changelog is updated
-- [ ] Tag 1 of @kyranjamie or @zone117x for review
+- [ ] Tag 1 of @rafaelcr or @zone117x for review

.github/workflows/gh-pages.yml

@@ -1,16 +1,22 @@
 name: gh-pages
 
 on:
-  push:
+  workflow_run:
+    workflows:
+      - "stacks-blockchain-api"
     branches:
-      - 'master'
+      - master
+    types:
+      - completed
   workflow_dispatch:
 
 jobs:
   gh-pages:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
       - name: Use Node.js
         uses: actions/setup-node@v2
@@ -26,15 +32,18 @@ jobs:
         run: npm run generate:docs
         working-directory: client
 
+      - name: Generate git info
+        run: npm run git-info
+
       - name: Install deps
         run: npm install
         working-directory: docs
 
-      - name: Generate and deploy gh-pages
-        run: npm run generate:types && npm run generate:docs && npm run generate:postman
+      - name: Generate docs
+        run: npm run generate:resolved-spec && npm run generate:types && npm run generate:docs && npm run generate:postman
         working-directory: docs
 
-      - name: Publish test coverage report
+      - name: Deploy gh-pages
         uses: JamesIves/[email protected]
         with:
           branch: gh-pages

.github/workflows/stacks-blockchain-api.yml

@@ -266,6 +266,126 @@ jobs:
         uses: codecov/codecov-action@v1
         if: always()
 
+  test-rosetta-cli-data:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Use Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+          check-latest: true
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: |
+            ~/.npm
+            node_modules
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Install deps
+        run: npm install --audit=false
+
+      - name: Setup env vars
+        run: echo "STACKS_CORE_EVENT_HOST=http://0.0.0.0">> $GITHUB_ENV
+
+      - name: Setup cli enviroment variable
+        run: | 
+          echo STACKS_BLOCKCHAIN_API_HOST=0.0.0.0>> .env
+          echo STACKS_CORE_PROXY_HOST=0.0.0.0 >> .env
+          echo STACKS_CORE_RPC_HOST=0.0.0.0 >> .env
+          echo STACKS_CORE_EVENT_HOST=0.0.0.0 >> .env 
+          echo BTC_RPC_HOST=http://0.0.0.0 >> .env 
+
+      - name: Setup integration environment
+        run: |
+          sudo ufw disable
+          npm run devenv:deploy -- -d
+          npm run devenv:logs -- --no-color &> docker-compose-logs.txt &
+
+      - name: Run tests
+        run: sudo npm run test:rosetta-cli:data
+
+      - name: Print integration environment logs
+        run: cat docker-compose-logs.txt
+        if: failure()
+
+      - name: Teardown integration environment
+        run: npm run devenv:stop
+        if: always()
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        if: always()
+
+  test-rosetta-cli-construction:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Use Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+          check-latest: true
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: |
+            ~/.npm
+            node_modules
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Install deps
+        run: npm install --audit=false
+
+      - name: Setup env vars
+        run: echo "STACKS_CORE_EVENT_HOST=http://0.0.0.0">> $GITHUB_ENV
+      
+      - name: Setup cli enviroment variable
+        run: | 
+          echo STACKS_BLOCKCHAIN_API_HOST=0.0.0.0>> .env
+          echo STACKS_CORE_PROXY_HOST=0.0.0.0 >> .env
+          echo STACKS_CORE_RPC_HOST=0.0.0.0 >> .env
+          echo STACKS_CORE_EVENT_HOST=0.0.0.0 >> .env 
+          echo BTC_RPC_HOST=http://0.0.0.0 >> .env 
+
+      - name: Setup integration environment
+        run: |
+          sudo ufw disable
+          npm run devenv:deploy -- -d
+          npm run devenv:logs -- --no-color &> docker-compose-logs.txt &
+
+      - name: Run tests
+        run: sudo npm run test:rosetta-cli:construction
+
+      - name: Print integration environment logs
+        run: cat docker-compose-logs.txt
+        if: failure()
+
+      - name: Teardown integration environment
+        run: npm run devenv:stop
+        if: always()
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        if: always()
+
   test-tokens:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -126,3 +126,7 @@ yarn.lock
 
 # Git info output
 .git-info
+
+#rosetta-cli results
+rosetta-output-construction/rosetta-cli-output-const.json
+rosetta-output/rosetta-cli-output.json

.vscode/launch.json

@@ -49,6 +49,34 @@
       },
       "killBehavior": "polite",
     },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Launch: read-only",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "runtimeArgs": [
+        "-r",
+        "ts-node/register/transpile-only",
+        "-r",
+        "tsconfig-paths/register"
+      ],
+      "args": [
+        "${workspaceFolder}/src/index.ts"
+      ],
+      "outputCapture": "std",
+      "internalConsoleOptions": "openOnSessionStart",
+      "env": {
+        "STACKS_BLOCKCHAIN_API_PORT": "3998",
+        "STACKS_READ_ONLY_MODE": "true",
+        "STACKS_BLOCKCHAIN_API_DB": "pg",
+        "STACKS_CHAIN_ID": "0x80000000",
+        "NODE_ENV": "development",
+        "TS_NODE_SKIP_IGNORE": "true"
+      },
+      "killBehavior": "polite",
+    },
     {
       "type": "node",
       "request": "launch",
@@ -182,5 +210,11 @@
       "cwd": "${workspaceFolder}/docs",
       "args": ["${workspaceFolder}/docs/scripts/generate-types.ts"]
     }
+  ],
+  "compounds": [
+    {
+      "name": "Launch: mocknet reader-writer",
+      "configurations": ["Launch: mocknet", "Launch: read-only"]
+    }
   ]
 }

.vscode/tasks.json

@@ -14,7 +14,7 @@
     {
       "label": "stacks-node:deploy-dev",
       "type": "shell",
-      "command": "npm run devenv:deploy -- -d && npm run devenv:logs",
+      "command": "npm run devenv:deploy -- -d",
       "isBackground": true,
       "problemMatcher": [
         {