Merge pull request #18 from jchen42703/fix/remove-robotjs

Fix/remove robotjs

README.md

@@ -3,19 +3,25 @@
 ![](https://github.com/jchen42703/ai_mouse_movements/blob/master/images/mouse_movement_mvp_mouseMove.gif)
 Send POST requests to automatically move your mouse with a neural network!
 
+(Note: The api now only generates coordinates and no longer moves the mouse for you.)
+
 ## Table of Contents
 
-- [`pymousegan`](#pymousegan)
-  - [Getting Started](#getting-started)
-  - [Dependencies](#dependencies)
-  - [Training Pipeline](#training-pipeline)
-    - [Preprocessing](#preprocessing)
-    - [GAN](#gan)
-- [JS API](#js-api)
-  - [Getting Started (Client)](#getting-started-client)
-  - [Model Format Conversion](#model-format-conversion)
-  - [Dependencies](#dependencies-1)
-  - [How does it work?](#how-does-it-work)
+- [API for Generating Mouse Movements with Neural Networks](#api-for-generating-mouse-movements-with-neural-networks)
+  - [Table of Contents](#table-of-contents)
+  - [pymousegan](#pymousegan)
+    - [Getting Started](#getting-started)
+    - [Dependencies](#dependencies)
+    - [Training Pipeline](#training-pipeline)
+      - [Preprocessing](#preprocessing)
+      - [GAN](#gan)
+  - [[JS API]](#js-api)
+    - [Getting Started [Client]](#getting-started-client)
+    - [Model Format Conversion](#model-format-conversion)
+      - [From `tf.keras` to `.json`](#from-tfkeras-to-json)
+    - [Dependencies](#dependencies-1)
+    - [How does it work?](#how-does-it-work)
+    - [How fast is the api?](#how-fast-is-the-api)
 
 ---
 
@@ -55,6 +61,7 @@ pip install .
 The model used in the current version is a `BidirectionalLSTMDecoderGenerator` from an `AdditiveBasicGAN` with a `BidirectionalLSTMDiscriminator` (with minibatch discrimination) and `BidirectionalLSTMDecoderGenerator`. The full example is located at https://github.com/jchen42703/ai_mouse_movements/python/README.md.
 
 Here are the model summaries:
+
 ![](images\model_summaries.png)
 
 ---
@@ -63,20 +70,23 @@ Here are the model summaries:
 
 ### Getting Started [Client]
 
+```
+cd js
+npm install .
+nodemon index.js
+```
+
 1. Install dependencies with `npm install`
 2. `nodemon index.js` or `node index.js` to run the server on `PORT=3000`.
 3. Send a `POST` request (`json`) to `http://localhost:3000/`, such as:
 
 ```
 {
     "start": [1, 1],
-    "destination": [82 ,55],
-    "moveMouse": 1
+    "destination": [82 ,55]
 }
 ```
 
-If you want a `json` response of the coords and lags, then do `"mouseMove": 0`.
-
 ![](images\mouseMove0.png)
 
 ### Model Format Conversion
@@ -93,12 +103,52 @@ tensorflowjs_converter --input_format=keras model/weights.h5 model/tfjs_model
 - `@tensorflow/tfjs`
 - `@tensorflow/tfjs-node`
 - `express`
-- `body-parser`
-- `robotjs` for mouse movements
 - `nodemon` for convenience
 
 ### How does it work?
 
 1. `POST` request to `https://localhost:3000/`
 2. `express` handles the `POST` request and calls the prediction function `loadAndPredict`.
-3. The function returns a promise, and the mouse movement (`robotjs`) resolved using this promise.
+3. The function returns a promise and when it resolves, the output is a list of coords and lags:
+
+- `[x, y, lag]`
+  - The `lag` is the time in `ms` that the mouse stays at that coordinate
+
+```
+{
+  "coords": [
+    [
+      1,
+      1,
+      24.451885223388672
+    ],
+    [
+      1.789207100868225,
+      1.6034066677093506,
+      23.39274024963379
+    ],
+    [
+      2.462282180786133,
+      2.276571035385132,
+      24.84036636352539
+    ],
+    [
+      2.7074904441833496,
+      2.716768264770508,
+      26.283510208129883
+    ],
+    [
+      2.862687110900879,
+      3.18359637260437,
+      27.842201232910156
+    ],
+    ...
+  ]
+}
+```
+
+### How fast is the api?
+
+On average, it runs from 390ms to 430ms
+
+- For cold starts: 500ms - 600ms

js/README.md

@@ -9,13 +9,10 @@
 ```
 {
     "start": [1, 1],
-    "destination": [82 ,55],
-    "moveMouse": 1
+    "destination": [82 ,55]
 }
 ```
 
-If you want a `json` response of the coords and lags, then do `"mouseMove": 0`.
-
 ## Model Conversion (`tf.keras` to `.json`)
 
 ```

js/index.js

@@ -1,17 +1,12 @@
 const predAPI = require("./src/predictAPI.js");
-const mouse = require("./src/mouse.js");
-const path = require("path");
-const bodyParser = require("body-parser");
 const express = require("express");
 const app = express();
 
 let relativeModelPath = "src/model/tfjs_model/model.json";
 
-app.use(bodyParser.json());
+app.use(express.json());
 
 app.post("/", (req, res) => {
-  // console.log(req.body);
-  // console.log(`dir: ${path.join(__dirname, relativeModelPath)}`);
   let api = new predAPI.predictAPI(
     relativeModelPath,
     req.body.start,
@@ -20,22 +15,10 @@ app.post("/", (req, res) => {
   const pred_promise = api.predict();
   pred_promise.then((pred_promise) => {
     var pred = pred_promise;
-    // console.log(`prediction: ${pred}, shape: ${pred.shape}`);
-    // move mouse....
-    if (req.body.moveMouse) {
-      mouse.moveMousePath(pred);
-      res.end("Movement done!");
-    } else {
-      const mouseJson = api.parseToJson(pred);
-      res.send(mouseJson);
-    }
+    const mouseJson = api.parseToJson(pred);
+    res.send(mouseJson);
   });
 });
 
-app.get("/", (req, res) => {
-  res.sendFile(path.join(__dirname, relativeModelPath));
-  res.end("File sent!");
-});
-
 const PORT = process.env.PORT || 3000;
 app.listen(PORT, () => console.log(`listening on ${PORT}`));