Improving Self-Hosting and Removing 3rd Party dependencies. (#4513)

* fix: Library Header layout shift

* Bump Github Actions versions.

* Self-Hosting Changes

* Fix Minio Environment Variable

* Just make pdfs successful, due to lack of PDFHandler

* Fix issue where flag was set wrong

* Added an NGINX Example file

* Add some documentation for self-hosting via Docker Compose

* Make some adjustments to Puppeteer due to failing sites.

* adjust timings

* Add start of Mail Service

* Fix Docker Files

* More email service stuff

* Add Guide to use Zapier for Email-Importing.

* Ensure that if no env is provided it uses the old email settings

* Add some instructions for self-hosted email

* Add SNS Endpoints for Mail Watcher

* Add steps and functionality for using SES and SNS for email

* Uncomment a few jobs.

* Added option for Firefox for parser. Was having issues with Chromium on Docker.

* Add missing space.

Co-authored-by: Russ Taylor <[email protected]>

* Fix some wording on the Guide

* update browser extension to handle self-hosted instances

* add slight documentation to options page

* Fix MV

* Do raw handlers for Medium

* Fix images in Medium

* Update self-hosting/GUIDE.md

Co-authored-by: Mike Baker <[email protected]>

* Update Guide with other variables

* Add The Verge to JS-less handlers

* Update regex and image-proxy

* Update self-hosting/nginx/nginx.conf

Co-authored-by: Mike Baker <[email protected]>

* Update regex and image-proxy

* Update self-hosting/docker-compose/docker-compose.yml

Co-authored-by: Mike Baker <[email protected]>

* Fix Minio for Export

* Merge to main

* Update GUIDE with newer NGINX

* Update nginx config to include api/save route

* Enable Native PDF View for PDFS

* Enable Native PDF View for PDFS

* feat:lover packages test

* feat:working build

* feat:alpine build

* docs:api dockerfile docs

* Write a PDF.js wrapper to replace pspdfkit

* Revert changes for replication, set settings to have default mode

* build folder got removed due to gitignore on pdf

* Add Box shadow to pdf pages

* Add Toggle for Progress in PDFS, enabled native viewer toggle

* Update node version to LTS

* Update node version to LTS

* Fix Linting issues

* Fix Linting issues

* Make env variable nullable

* Add touchend listener for mobile

* Make changes to PDF for mobile

* fix(android): change serverUrl to selfhosted first

* feat:2 stage alpine content fetch

* feat:separated start script

* fix:changed to node 22

* Add back youtube functionality and add guide

* trigger build

* Fix cache issue on YouTube

* Allow empty AWS_S3_ENDPOINT

* Allow empty AWS_S3_ENDPOINT

* Add GCHR for all images

* Add GCHR For self hosting.

* Add GCHR For self hosting.

* Test prebuilt.

* Test prebuilt

* Test prebuilt...

* Fix web image

* Remove Web Image (For now)

* Move docker-compose to images

* Move docker-compose files to correct locations

* Remove the need for ARGS

* Update packages, and Typescript versions

* Fix

* Fix issues with build on Web

* Correct push

* Fix Linting issues

* Fix Trace import

* Add missing types

* Fix Tasks

* Add information into guide about self-build

* Fix issues with PDF Viewer

---------

Co-authored-by: keumky2 <[email protected]>
Co-authored-by: William Theaker <[email protected]>
Co-authored-by: Russ Taylor <[email protected]>
Co-authored-by: David Adams <[email protected]>
Co-authored-by: Mike Baker <[email protected]>
Co-authored-by: m1xxos <[email protected]>
Co-authored-by: Adil <[email protected]>

.eslintrc

@@ -11,6 +11,7 @@
   },
   "plugins": ["@typescript-eslint"],
   "rules": {
-    "semi": [2, "never"]
+    "semi": [2, "never"],
+    "@typescript-eslint/no-unnecessary-type-assertion": [0, "never"]
   }
 }

.github/workflows/build-docker-images.yml

@@ -13,11 +13,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: 'Login to GitHub container registry'
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{github.actor}}
@@ -30,3 +30,4 @@ jobs:
         run: |
           docker build --file packages/content-fetch/Dockerfile . --tag "ghcr.io/omnivore-app/content-fetch:${GITHUB_SHA}" --tag ghcr.io/omnivore-app/content-fetch:latest
           docker push ghcr.io/omnivore-app/content-fetch:${GITHUB_SHA}
+   

.github/workflows/build-self-host-docker-images.yml

@@ -0,0 +1,64 @@
+name: Build Self-Hosting Docker Images
+on:
+  push:
+    branches:
+      - main
+      - self-host-updates
+    paths-ignore:
+      - 'apple/**'
+      - 'android/**'
+
+jobs:
+  build-self-hostdocker-images:
+    name: Build self-host docker images
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: 'Login to GitHub container registry'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{github.actor}}
+          password: ${{secrets.GITHUB_TOKEN}}
+      - name: Build the backend docker image
+        run: |
+          docker build . --file packages/api/Dockerfile --tag "ghcr.io/omnivore-app/sh-backend:${GITHUB_SHA}" --tag ghcr.io/omnivore-app/sh-backend:latest
+          docker push ghcr.io/omnivore-app/sh-backend:${GITHUB_SHA}
+          docker push ghcr.io/omnivore-app/sh-backend:latest
+      - name: Build the content-fetch docker image
+        run: |
+          docker build --file packages/content-fetch/Dockerfile . --tag "ghcr.io/omnivore-app/sh-content-fetch:${GITHUB_SHA}" --tag ghcr.io/omnivore-app/sh-content-fetch:latest
+          docker push ghcr.io/omnivore-app/sh-content-fetch:${GITHUB_SHA}
+          docker push ghcr.io/omnivore-app/sh-content-fetch:latest
+      - name: Build the queue-processor docker image.
+        run: |
+          docker build . --file packages/api/queue-processor/Dockerfile --tag "ghcr.io/omnivore-app/sh-queue-processor:${GITHUB_SHA}" --tag ghcr.io/omnivore-app/sh-queue-processor:latest
+          docker push ghcr.io/omnivore-app/sh-queue-processor:${GITHUB_SHA}
+          docker push ghcr.io/omnivore-app/sh-queue-processor:latest
+      - name: Build the migrate docker image
+        run: |
+          docker build --file packages/db/Dockerfile . --tag "ghcr.io/omnivore-app/sh-migrate:${GITHUB_SHA}" --tag ghcr.io/omnivore-app/sh-migrate:latest
+          docker push ghcr.io/omnivore-app/sh-migrate:${GITHUB_SHA}
+          docker push ghcr.io/omnivore-app/sh-migrate:latest
+      - name: Build the image-proxy docker image
+        run: |
+          cp imageproxy/start_imageproxy.sh .
+          chmod +x start_imageproxy.sh
+          docker build --file imageproxy/Dockerfile . --tag "ghcr.io/omnivore-app/sh-image-proxy:${GITHUB_SHA}" --tag ghcr.io/omnivore-app/sh-image-proxy:latest
+          docker push ghcr.io/omnivore-app/sh-image-proxy:${GITHUB_SHA}
+          docker push ghcr.io/omnivore-app/sh-image-proxy:latest
+      - name: Build the mail-watch-server docker image
+        run: |
+          docker build --file packages/local-mail-watcher/Dockerfile . --tag "ghcr.io/omnivore-app/sh-local-mail-watcher:${GITHUB_SHA}" --tag ghcr.io/omnivore-app/sh-local-mail-watcher:latest
+          docker push ghcr.io/omnivore-app/sh-local-mail-watcher:${GITHUB_SHA}
+          docker push ghcr.io/omnivore-app/sh-local-mail-watcher:latest
+    
+

.github/workflows/lint-migrations.yml

@@ -9,7 +9,7 @@ jobs:
   lint_migrations:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
       - name: Fetch main branch
         run: git fetch origin main:main
       - name: Find modified migrations

.github/workflows/run-distiller.yml

@@ -14,12 +14,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: ${{ github.event.pull_request.head.repo.full_name }}
           ref: ${{ github.event.pull_request.head.ref }}
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
           go-version: 1.19
       - name: Generate distiller output for readability
@@ -34,7 +34,7 @@ jobs:
             go-domdistiller file -i $f/source.html -o $f/distiller.html
           done
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
       - name: Generate static html

.github/workflows/run-tests.yaml

@@ -45,7 +45,7 @@ jobs:
         ports:
           - 6379
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Initialize the database
@@ -55,9 +55,9 @@ jobs:
         env:
           PGPASSWORD: postgres
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v4
         with:
-          node-version: 18.16
+          node-version: 22.12.0
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(source ~/.nvm/nvm.sh && yarn cache dir)"
@@ -102,7 +102,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Build the API docker image

.github/workflows/static.yml

@@ -26,13 +26,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup Pages
-        uses: actions/configure-pages@v3
+        uses: actions/configure-pages@v5
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v1
+        uses: actions/upload-pages-artifact@v3
         with:
           path: 'packages/readabilityjs/test'
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v4

.node-version

@@ -1 +1 @@
-18.16
+22.12.0

README.md

@@ -151,24 +151,7 @@ is done fetching your content you will see it in your library.
 
 ## How to deploy to your own server
 
-Omnivore was originally designed to be deployed on GCP and takes advantage
-of some of GCP's PaaS features. We are working to make Omnivore more portable
-so you can easily run the service on your own infrastructure. You can track
-progress here: <https://github.com/omnivore-app/omnivore/issues/25>
-
-To deploy Omnivore on your own hardware you will need to deploy three
-dockerized services and configure access to a postgres service. To handle
-PDF documents you will need to configure access to a Google Cloud Storage
-bucket.
-
-- `packages/api` - the backend API service
-- `packages/web` - the web frontend (can easily be deployed to vercel)
-- `packages/puppeteer-parse` - the content fetching service (can easily
-  be deployed as an AWS lambda or GCP Cloud Function)
-
-Additionally, you will need to run our database migrations to initialize
-your database. These are dockerized and can be run with the
-`packages/db` service.
+A guide for running a self hosted server can be found [here](./self-hosting/GUIDE.md)
 
 ## License
 

android/Omnivore/README.md

@@ -1,5 +1,12 @@
 # Omnivore - Android
 
+## Setup with gradle
+
+In case you do not have Android Studio and you do not want to install it, you may want to use gradlew scripts to build the application.
+
+`./gradlew assembleDebug` should create `./app/build/outputs/apk/debug/app-debug.apk`
+
+
 ## Setup
 
 From the root directory run the following command:

android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/editinfo/EditInfoViewModel.kt

@@ -10,6 +10,7 @@ import app.omnivore.omnivore.R
 import app.omnivore.omnivore.core.data.DataService
 import app.omnivore.omnivore.core.datastore.DatastoreRepository
 import app.omnivore.omnivore.core.datastore.omnivoreAuthToken
+import app.omnivore.omnivore.core.datastore.omnivoreSelfHostedApiServer
 import app.omnivore.omnivore.graphql.generated.UpdatePageMutation
 import app.omnivore.omnivore.graphql.generated.type.UpdatePageInput
 import app.omnivore.omnivore.utils.Constants
@@ -48,6 +49,12 @@ class EditInfoViewModel @Inject constructor(
     datastoreRepo.getString(omnivoreAuthToken)
   }
 
+  fun baseUrl() = runBlocking {
+        datastoreRepo.getString(omnivoreSelfHostedApiServer) ?: Constants.apiURL
+  }
+
+  private fun serverUrl() = "${baseUrl()}/api/graphql"
+
   fun editInfo(itemId: String, title: String, author: String?, description: String?) {
     viewModelScope.launch {
       isLoading = true
@@ -62,7 +69,7 @@ class EditInfoViewModel @Inject constructor(
       }
 
       val apolloClient = ApolloClient.Builder()
-        .serverUrl("${Constants.apiURL}/api/graphql")
+        .serverUrl(serverUrl())
         .addHttpHeader("Authorization", value = authToken)
         .build()
 

android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/onboarding/OnboardingViewModel.kt

@@ -176,6 +176,12 @@ class OnboardingViewModel @Inject constructor(
         resetPendingEmailUserCreds()
     }
 
+    fun baseUrl() = runBlocking {
+        datastoreRepository.getString(omnivoreSelfHostedApiServer) ?: Constants.apiURL
+    }
+
+    private fun serverUrl() = "${baseUrl()}/api/graphql"
+
     fun validateUsername(potentialUsername: String) {
         validateUsernameJob?.cancel()
 
@@ -209,7 +215,7 @@ class OnboardingViewModel @Inject constructor(
             }
 
             val apolloClient =
-                ApolloClient.Builder().serverUrl("${Constants.apiURL}/api/graphql").build()
+                ApolloClient.Builder().serverUrl(serverUrl()).build()
 
             try {
                 val response = apolloClient.query(

android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/save/SaveURLWorker.kt

@@ -16,6 +16,7 @@ import androidx.work.WorkerParameters
 import app.omnivore.omnivore.R
 import app.omnivore.omnivore.core.datastore.DatastoreRepository
 import app.omnivore.omnivore.core.datastore.omnivoreAuthToken
+import app.omnivore.omnivore.core.datastore.omnivoreSelfHostedApiServer
 import app.omnivore.omnivore.graphql.generated.SaveUrlMutation
 import app.omnivore.omnivore.graphql.generated.type.SaveUrlInput
 import app.omnivore.omnivore.utils.Constants
@@ -59,11 +60,16 @@ class SaveURLWorker @AssistedInject constructor(
         }
     }
 
+    suspend fun baseUrl() =
+        datastoreRepository.getString(omnivoreSelfHostedApiServer) ?: Constants.apiURL
+
+    private suspend fun serverUrl() = "${baseUrl()}/api/graphql"
+
     private suspend fun saveURL(url: String): Boolean {
         val authToken = datastoreRepository.getString(omnivoreAuthToken) ?: return false
 
         val apolloClient = ApolloClient.Builder()
-            .serverUrl("${Constants.apiURL}/api/graphql")
+            .serverUrl(serverUrl())
             .addHttpHeader("Authorization", value = authToken)
             .build()
 

android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/save/SaveViewModel.kt

@@ -11,6 +11,7 @@ import androidx.lifecycle.viewModelScope
 import app.omnivore.omnivore.R
 import app.omnivore.omnivore.core.datastore.DatastoreRepository
 import app.omnivore.omnivore.core.datastore.omnivoreAuthToken
+import app.omnivore.omnivore.core.datastore.omnivoreSelfHostedApiServer
 import app.omnivore.omnivore.graphql.generated.SaveUrlMutation
 import app.omnivore.omnivore.graphql.generated.type.SaveUrlInput
 import app.omnivore.omnivore.utils.Constants
@@ -71,6 +72,12 @@ class SaveViewModel @Inject constructor(
     return null
   }
 
+  fun baseUrl() = runBlocking {
+        datastoreRepo.getString(omnivoreSelfHostedApiServer) ?: Constants.apiURL
+  }
+
+  private fun serverUrl() = "${baseUrl()}/api/graphql"
+
   fun saveURL(url: String) {
     viewModelScope.launch {
       isLoading = true
@@ -86,7 +93,7 @@ class SaveViewModel @Inject constructor(
       }
 
       val apolloClient = ApolloClient.Builder()
-        .serverUrl("${Constants.apiURL}/api/graphql")
+        .serverUrl(serverUrl())
         .addHttpHeader("Authorization", value = authToken)
         .build()
 

imageproxy/Dockerfile

@@ -1,4 +1,4 @@
-FROM willnorris/imageproxy:v0.10.0 as build
+FROM ghcr.io/willnorris/imageproxy:main as build
 
 # Above imageproxy image is built from scratch image and is barebones
 # Switching over to ubuntu base image to allow us to debug better.

package.json

@@ -33,7 +33,7 @@
     "graphql-tag": "^2.11.0",
     "lerna": "^7.4.1",
     "prettier": "^2.5.1",
-    "typescript": "4.5.2"
+    "typescript": "5.7.3"
   },
   "volta": {
     "node": "18.16.1",

packages/api/.eslintrc

@@ -4,13 +4,27 @@
     "project": "tsconfig.json"
   },
   "rules": {
-    "@typescript-eslint/no-unsafe-argument": 0
+    "@typescript-eslint/no-unsafe-argument": 0,
+    "@typescript-eslint/no-unsafe-member-access": 0,
+    "@typescript-eslint/no-unnecessary-type-assertion": 0,
+    "@typescript-eslint/no-unsafe-assignment": 0,
+    "@typescript-eslint/no-unsafe-return": ["warn"],
+    "@typescript-eslint/no-unsafe-call" : ["warn"],
+    "@typescript-eslint/restrict-template-expressions": ["warn"],
+    "@typescript-eslint/no-misused-promises": ["warn"],
+    "@typescript-eslint/ban-ts-comment": ["warn"],
+    "@typescript-eslint/no-unused-vars": 0
   },
   "overrides": [
     {
       "files": ["test/**/*.ts"],
       "rules": {
-        "@typescript-eslint/no-unsafe-member-access": 0
+        "@typescript-eslint/no-unsafe-member-access": 0,
+        "@typescript-eslint/no-unnecessary-type-assertion": 0,
+        "@typescript-eslint/no-unsafe-assignment": 0,
+        "@typescript-eslint/no-unsafe-return": ["warn"],
+        "@typescript-eslint/no-unsafe-call" : ["warn"],
+        "@typescript-eslint/no-unused-vars": 0
       }
     }
   ]