fixes

Author: Chetan11-dev

.erb/configs/webpack-watch-files-plugin.ts

@@ -0,0 +1,85 @@
+import glob from 'glob';
+import { resolve } from 'path';
+import touch from 'touch';
+import type { Compiler, Compilation } from 'webpack';
+
+interface WebpackWatchPluginOptions extends Omit<any, 'absolute'> {
+  files?: string[];
+  verbose?: boolean;
+}
+
+class WebpackWatchPlugin {
+  private files: string[];
+  private verbose: boolean;
+  private globOptions: any;
+  private filesAlreadyAdded: boolean;
+
+  constructor(options: WebpackWatchPluginOptions = {}) {
+    const { files = [], verbose, ...globOptions } = options;
+
+    this.files = files;
+    this.verbose = !!verbose;
+    this.globOptions = {
+      absolute: true,
+      ...globOptions,
+    };
+    this.filesAlreadyAdded = false;
+  }
+
+  apply(compiler: Compiler): void {
+    compiler.hooks.afterCompile.tapAsync(
+      'WebpackWatchPlugin',
+      (compilation: Compilation, callback: () => void) => {
+        const filesFound: string[] = [];
+        const filesFoundToExclude: string[] = [];
+
+        for (const pattern of this.files) {
+          if (pattern.substr(0, 1) !== '!') {
+            glob.sync(pattern, this.globOptions).forEach((file) => {
+              filesFound.push(file);
+            });
+          } else {
+            glob.sync(pattern.substr(1), this.globOptions).forEach((file) => {
+              filesFoundToExclude.push(file);
+            });
+          }
+        }
+
+        const files = (
+          (
+            filesFound.map((file) => {
+              if (filesFoundToExclude.indexOf(file) !== -1) {
+                return null;
+              }
+              return file;
+            })
+          )
+        ).filter((file) => file !== null).map((file) => resolve(file));
+
+        if (this.verbose && !this.filesAlreadyAdded) {
+          console.log('Additional files watched:', JSON.stringify(files, null, 2));
+        }
+        console.log({ files });
+
+        files.forEach((file) => {
+          compilation.fileDependencies.add(file);
+        });
+
+        this.filesAlreadyAdded = true;
+
+        const outputPath = compilation.outputOptions.path;
+        const assets = compilation.getAssets();
+        if (assets.length > 0 && outputPath) {
+          const firstAssetName = assets[0].name;
+          const assetToTouch = resolve(outputPath, firstAssetName);
+          touch(assetToTouch);
+        }
+
+        callback();
+      }
+    );
+  }
+}
+
+export default WebpackWatchPlugin;
+

.erb/configs/webpack.config.main.dev.ts

@@ -11,22 +11,7 @@ import webpackPaths from './webpack.paths';
 import { devMainCopyPlugins } from './copy-plugin'
 import { GenerateApiPropsPlugin } from './webpack.custom-plugins'
 import { getAnalyzerPlugins } from './getAnalyzerPlugins'
-import WatchExternalFilesPlugin from 'webpack-watch-files-plugin'
-import touch from 'touch';
-
-
-class WebpackForceRebuildOnEmitPlugin {
-  apply(compiler) {
-    compiler.hooks.emit.tapAsync('WebpackForceRebuildOnEmitPlugin', (compilation, callback) => {
-      const outputPath = compilation.outputOptions.path;
-      const firstAssetName = compilation.getAssets()[0].name;
-      const assetToTouch = path.resolve(outputPath, firstAssetName);
-      touch(assetToTouch);
-      callback();
-    });
-  }
-}
-
+import WatchExternalFilesPlugin from './webpack-watch-files-plugin'
 
 function getWatchingPlugins() {
   const IS_WATCH_MODE = process.argv?.includes('--watch');
@@ -36,9 +21,8 @@ function getWatchingPlugins() {
       verbose: false,
       files: [
         './inputs/**/*.js',
-      ]
+      ], 
     }) as any,
-    new WebpackForceRebuildOnEmitPlugin(),
   ] : []
 }
 // When an ESLint server is running, we can't set the NODE_ENV so we'll check if it's

Dockerfile

@@ -3,24 +3,6 @@ FROM node:20
 ENV DEBIAN_FRONTEND=noninteractive \
     DISPLAY=:99
 
-# Install system dependencies
-# RUN apt-get update && apt-get install -y --no-install-recommends \
-#     wget \
-#     git \
-#     lsof \
-#     xvfb \
-#     gnupg \
-#     ca-certificates \
-#     && rm -rf /var/lib/apt/lists/*
-
-# Install Chromium (works on both AMD64 and ARM64)
-# RUN apt-get update \
-#     && apt-get install -y --no-install-recommends chromium \
-#     && rm -rf /var/lib/apt/lists/* \
-#     && ln -sf /usr/bin/chromium /usr/bin/google-chrome-stable \
-#     && ln -sf /usr/bin/chromium /usr/bin/google-chrome
-
-
 # # Install Chrome dependencies
 RUN apt-get update && apt-get install -y \
     wget \
@@ -44,8 +26,12 @@ RUN apt-get update && apt-get install -y google-chrome-stable \
 WORKDIR /app
 
 COPY package*.json ./
-COPY . .
+COPY tsconfig.json ./
+COPY release/ ./release/
+COPY .erb/ ./.erb/
+
 RUN npm install
+COPY . .
 
 
 EXPOSE 8000

package-lock.json

@@ -29,7 +29,7 @@
     "clean:db": "rimraf output/ error_logs/ task_results/ db.nedb",
     "update": "npm update botasaurus botasaurus-controls botasaurus-server",
     "start:main": "concurrently -k \"cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack --watch --config ./.erb/configs/webpack.config.main.dev.ts\" \"electronmon .\"",
-    "start:main:worker": "concurrently -k \"cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack --watch --config ./.erb/configs/webpack.config.main.dev.ts\" \"electronmon . --worker\"",
+    "start:main:worker": "concurrently -k \"cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack --watch --config ./.erb/configs/webpack.config.main.dev.ts\" \"electronmon . --worker --no-sandbox\"",
     "start:main:master": "concurrently -k \"cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack --watch --config ./.erb/configs/webpack.config.main.dev.ts\" \"electronmon . --master\"",
     "start:preload": "cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack --config ./.erb/configs/webpack.config.preload.dev.ts",
     "start:renderer": "cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack serve --config ./.erb/configs/webpack.config.renderer.dev.ts",
@@ -139,8 +139,7 @@
     "webpack-bundle-analyzer": "^4.9.1",
     "webpack-cli": "^5.1.4",
     "webpack-dev-server": "^4.15.1",
-    "webpack-merge": "^5.9.0",
-    "webpack-watch-files-plugin": "^1.2.1"
+    "webpack-merge": "^5.9.0"
   },
   "build": {
     "productName": "Todo My App Name",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "todo-my-app-name",
-  "version": "1.40.0",
+  "version": "1.41.0",
   "description": "Todo my app description",
   "license": "MIT",
   "main": "./dist/main/main.js",

release/app/package-lock.json

@@ -23,7 +23,7 @@ export function addRoutesHandler() {
           return pickId(result)
         }
 
-        return result.slice(0, 1).map(pickId)
+        return [pickId(result[1] ?? result[0])]
       })
     } else {
       // @ts-ignore

release/app/package.json

@@ -27,6 +27,7 @@ import SearchSingleSelectApi from '../inputs/SearchSingleSelectApi';
 import SearchMultiSelectApi from '../inputs/SearchMultiSelectApi';
 import SwitchField from '../inputs/SwitchField';
 import TextAreaField from '../inputs/TextAreaField';
+import JsonTextAreaField from '../inputs/JsonTextAreaField';
 import TextField from '../inputs/TextField';
 import { useRouter } from '../Link';
 import ScraperSelector from '../ScraperSelector/ScraperSelector';
@@ -175,6 +176,18 @@ const InputFields = ({
               />
             )
             break
+          case 'jsonTextArea':
+            inputElement = (
+              <JsonTextAreaField
+                title={disabledMsg}
+                disabled={disabled}
+                placeholder={(control as any).placeholder}
+                name={id}
+                value={data[id]}
+                onChange={value => handleInputChange(id, value)}
+              />
+            )
+            break
           case 'number':
             inputElement = (
               <NumberField
@@ -537,13 +550,11 @@ const ScraperFormContainer = ({ scrapers, enable_cache }) => {
     e?.preventDefault()
     setSubmitAttempted(true)
     if (isEmptyObject(validationResult)) {
-      // @ts-ignore
-      const cleanedData = controls.getBackendValidationResult(data)['data']
       setIsSubmitting(true)
       try {
         const response = await Api.createAsyncTask({
           scraper_name: selectedScraper.scraper_name,
-          data: cleanedData,
+          data,
           enable_cache: enableCache}).finally(() => setIsSubmitting(false))
         if (response) {
           const result = response.data