fix: LSDV-4692: Brush segmentation is not supported (#1247)

* Fix tools in MIG

* Remove excess FF

* Make RLE Export stageless

* Remove feature flag

* Fix comment

* Remove unused helper

* Remove unused code

* fix: LSDV-4774: Drawing on the hidden image in MIG scenario (#1254)

Fixing Brush and Polygon can draw on hiddent images

* Fix inability to edit brush regions

* Remove RLE cache

* Fix helper

* Rename test suite

* Remove ESLint debug info

* Fix audio assertions

* Fix audio error matching

---------

Co-authored-by: Sergey <[email protected]>

Author: nick-skriabin

e2e/fragments/AtAudioView.js

@@ -277,8 +277,9 @@ module.exports = {
   async seeErrorHandler(value, selector = null) {
     selector = selector ? this[selector] : this._errorSelector;
     const error = await I.grabTextFrom(selector);
+    const matcher = new RegExp(value);
 
-    assert.equal(error, value);
+    assert.match(error, matcher);
   },
 
   /**

e2e/fragments/AtImageView.js

@@ -50,9 +50,9 @@ module.exports = {
     return this._stageBBox?.y ?? 0;
   },
 
-  waitForImage() {
+  async waitForImage() {
     I.say('Waiting for image to be loaded');
-    I.executeScript(Helpers.waitForImage);
+    await I.executeScript(Helpers.waitForImage);
     I.waitForVisible('canvas', 5);
   },
 
@@ -300,4 +300,18 @@ module.exports = {
     I.say('Select move tool');
     I.pressKey('V');
   },
+
+  async multiImageGoForwartWithHotkey() {
+    I.say('Attempting to go to the next image');
+    I.pressKey('Ctrl+d');
+
+    await this.waitForImage();
+  },
+
+  async multiImageGoBackwardWithHotkey() {
+    I.say('Attempting to go to the next image');
+    I.pressKey('Ctrl+a');
+
+    await this.waitForImage();
+  },
 };

e2e/tests/audio/audio-errors.test.js

@@ -55,7 +55,7 @@ Scenario('Check if audio decoder error handler is showing', async function({ I,
 
   await AtAudioView.lookForStage();
 
-  AtAudioView.seeErrorHandler('An error occurred while decoding the audio file');
+  await AtAudioView.seeErrorHandler('An error occurred while decoding the audio file');
 });
 
 Scenario('Check if audio http error handler is showing', async function({ I, LabelStudio, AtAudioView }) {
@@ -74,5 +74,5 @@ Scenario('Check if audio http error handler is showing', async function({ I, Lab
 
   await AtAudioView.lookForStage();
 
-  AtAudioView.seeErrorHandler('HTTP error status: 404', '_httpErrorSelector');
+  await AtAudioView.seeErrorHandler('HTTP error status: 404', '_httpErrorSelector');
 });

e2e/tests/image-list.test.js

@@ -1,6 +1,8 @@
-Feature('Image list via `valueList`');
+Feature('MIG');
 
-const config = `
+const assert = require('assert');
+
+const rectConfig = `
   <View>
     <Image name="img" valueList="$images"/>
     <RectangleLabels name="tag" toName="img">
@@ -10,6 +12,16 @@ const config = `
   </View>
 `;
 
+const brushConfig = `
+  <View>
+    <Image name="img" valueList="$images"/>
+    <BrushLabels name="tag" toName="img">
+      <Label value="Planet"></Label>
+      <Label value="Moonwalker" background="blue"></Label>
+    </BrushLabels>
+  </View>
+`;
+
 const data = {
   images: [
     'https://data.heartex.net/open-images/train_0/mini/0030019819f25b28.jpg',
@@ -93,7 +105,7 @@ Before(async ({ LabelStudio }) => {
 
 Scenario('Image list rendering', async ({ I, LabelStudio, AtImageView }) => {
   const params = {
-    config,
+    config: rectConfig,
     data,
     annotations: [{ id: 1, result: [] }],
   };
@@ -109,7 +121,7 @@ Scenario('Image list rendering', async ({ I, LabelStudio, AtImageView }) => {
 
 Scenario('Image list with page navigation', async ({ I, AtImageView, LabelStudio }) => {
   const params = {
-    config,
+    config: rectConfig,
     data,
     annotations: [{ id: 1, result: [] }],
   };
@@ -147,7 +159,7 @@ Scenario('Image list with page navigation', async ({ I, AtImageView, LabelStudio
 
 Scenario('Image list with hotkey navigation', async ({ I, AtImageView, LabelStudio }) => {
   const params = {
-    config,
+    config: rectConfig,
     data,
     annotations: [{ id: 1, result: [] }],
   };
@@ -167,15 +179,13 @@ Scenario('Image list with hotkey navigation', async ({ I, AtImageView, LabelStud
   I.say('The number of pages is correct');
   I.see('1 of 4');
 
-  I.say('Clicking on the next page');
-  I.pressKey('Ctrl+d');
+  await AtImageView.multiImageGoForwartWithHotkey();
 
   I.say('Loading second image');
   I.seeElement(`img[src="${data.images[1]}"]`);
   I.see('2 of 4');
 
-  I.say('Clicking on the previous page');
-  I.pressKey('Ctrl+a');
+  await AtImageView.multiImageGoBackwardWithHotkey();
   I.seeElement(`img[src="${data.images[0]}"]`);
   I.see('1 of 4');
 });
@@ -186,7 +196,7 @@ Scenario('Ensure that results are the same when exporting existing regions', asy
   });
 
   const params = {
-    config,
+    config: rectConfig,
     data,
     annotations: [{ id: 1, result }],
   };
@@ -207,7 +217,7 @@ Scenario('Image list exports correct data', async ({ I, LabelStudio, AtImageView
   });
 
   const params = {
-    config,
+    config: rectConfig,
     data,
     annotations: [{ id: 1, result }],
   };
@@ -218,8 +228,7 @@ Scenario('Image list exports correct data', async ({ I, LabelStudio, AtImageView
   await AtImageView.waitForImage();
   await AtImageView.lookForStage();
 
-  I.say('Attempting to go to the next image');
-  I.pressKey('Ctrl+d');
+  AtImageView.multiImageGoForwartWithHotkey();
 
   await AtImageView.waitForImage();
   await AtImageView.lookForStage();
@@ -231,7 +240,7 @@ Scenario('Image list exports correct data', async ({ I, LabelStudio, AtImageView
 // TODO: temporarily disable, will be fixed in another ticket
 Scenario('Regions are not changes when duplicating an annotation', async ({ I, LabelStudio, AtImageView }) => {
   const params = {
-    config,
+    config: rectConfig,
     data,
     annotations: [{ id: 1, result }],
   };
@@ -251,3 +260,41 @@ Scenario('Regions are not changes when duplicating an annotation', async ({ I, L
   I.say('Confirm that result is not changed');
   await LabelStudio.resultsNotChanged(result);
 });
+
+Scenario('No errors during brush export in MIG', async ({ I, LabelStudio, AtImageView, AtLabels }) => {
+  const params = {
+    config: brushConfig,
+    data,
+    annotations: [{ id: 1, result: [] }],
+  };
+
+  const brushRegionPoints = [
+    [20, 20],
+    [20, 40],
+    [40, 40],
+    [40, 20],
+    [20, 20],
+  ];
+
+  I.amOnPage('/');
+  LabelStudio.init(params);
+
+  await AtImageView.waitForImage();
+  await AtImageView.lookForStage();
+
+  I.say('Create brush regions on the first image');
+  AtLabels.clickLabel('Moonwalker');
+  AtImageView.drawThroughPoints(brushRegionPoints);
+
+  await AtImageView.multiImageGoForwartWithHotkey();
+
+  I.pressKey('u');
+  I.say('Create brush regions on the second image');
+  AtLabels.clickLabel('Planet');
+  AtImageView.drawThroughPoints(brushRegionPoints);
+
+  const result = await LabelStudio.serialize();
+
+  assert.equal(result.length, 2);
+});
+

package.json

@@ -36,7 +36,7 @@
     "copy-examples": "bash scripts/copy.sh",
     "docs:generate": "node scripts/create-docs.js",
     "lint:write": "eslint --debug src/ --fix",
-    "lint": "eslint --debug src/",
+    "lint": "eslint src/",
     "prepublishOnly": "yarn run test && yarn run build:final",
     "prettier-styles": "prettier --write src/**/*.{css,scss}",
     "prettier": "prettier --write src/**/*.js",

src/components/ImageView/ImageView.js

@@ -86,6 +86,10 @@ const Regions = memo(({ regions, useLayers = true, chunkSize = 15, suggestion =
 
 const DrawingRegion = observer(({ item }) => {
   const { drawingRegion } = item;
+  
+  if (!drawingRegion) return null;
+  if (item.multiImage && item.currentImage !== drawingRegion.item_index) return null;
+
   const Wrapper = drawingRegion && drawingRegion.type === 'brushregion' ? Fragment : Layer;
 
   return (

src/mixins/DrawingTool.js

@@ -3,7 +3,7 @@ import { types } from 'mobx-state-tree';
 import Utils from '../utils';
 import throttle from 'lodash.throttle';
 import { MIN_SIZE } from '../tools/Base';
-import { FF_DEV_3666, FF_DEV_3793, FF_LSDV_4583, isFF } from '../utils/feature-flags';
+import { FF_DEV_3666, FF_DEV_3793, isFF } from '../utils/feature-flags';
 
 const DrawingTool = types
   .model('DrawingTool', {
@@ -111,8 +111,6 @@ const DrawingTool = types
         self.currentArea = self.obj.createDrawingRegion(opts, resultValue, control, false);
         self.currentArea.setDrawing(true);
 
-        if (isFF(FF_LSDV_4583)) self.currentArea.setItemIndex?.(self.obj.currentImage);
-
         self.applyActiveStates(self.currentArea);
         self.annotation.setIsDrawing(true);
         return self.currentArea;
@@ -138,8 +136,6 @@ const DrawingTool = types
 
         const newArea = self.annotation.createResult(value, currentArea.results[0].value.toJSON(), control, obj);
 
-        if (isFF(FF_LSDV_4583)) newArea.setItemIndex?.(obj.currentImage);
-
         currentArea.setDrawing(false);
         self.applyActiveStates(newArea);
         self.deleteRegion();
@@ -347,6 +343,11 @@ const MultipleClicksDrawingTool = DrawingTool.named('MultipleClicksMixin')
         return Super.canStartDrawing() && !self.annotation.regionStore.hasSelection;
       },
       nextPoint(x, y) {
+        const area = self.getCurrentArea();
+        const object = self.obj;
+
+        if (area && object && object.multiImage && area.item_index !== object.currentImage) return;
+
         self.getCurrentArea().addPoint(x, y);
         pointsCount++;
       },

src/regions/BrushRegion.js

@@ -64,8 +64,8 @@ const Points = types
   }))
   .actions(self => {
     return {
-      updateImageSize(wp, hp,sw,sh) {
-        self.points = self.relativePoints.map((v,idx) => {
+      updateImageSize(wp, hp, sw, sh) {
+        self.points = self.relativePoints.map((v, idx) => {
           const isX = !(idx % 2);
           const stageSize = isX ? sw : sh;
 
@@ -346,7 +346,7 @@ const Model = types
         annotation.autosave && setTimeout(() => annotation.autosave());
       },
 
-      convertPointsToMask() {},
+      convertPointsToMask() { },
 
       setScale(x, y) {
         self.scaleX = x;
@@ -497,14 +497,14 @@ const HtxBrushView = ({ item }) => {
     //  that dynamically produce image masks.
 
     if (!item.rle && !item.maskDataURL) return;
-    if (!item.parent || item.parent.naturalWidth <=1 || item.parent.naturalHeight <= 1) return;
+    if (!item.parent || item.parent.naturalWidth <= 1 || item.parent.naturalHeight <= 1) return;
 
     let img;
 
     if (item.maskDataURL && isFF(FF_DEV_4081)) {
       img = await Canvas.maskDataURL2Image(item.maskDataURL, { color: item.strokeColor });
     } else if (item.rle) {
-      img = Canvas.RLE2Region(item.rle, item.parent, { color: item.strokeColor });
+      img = Canvas.RLE2Region(item, { color: item.strokeColor });
     }
 
     if (img) {
@@ -689,8 +689,8 @@ const HtxBrushView = ({ item }) => {
           <Image
             name="highlight"
             image={highlightedImageRef.current}
-            sceneFunc={highlightedRef.current.highlighted ? null : () => {}}
-            hitFunc={() => {}}
+            sceneFunc={highlightedRef.current.highlighted ? null : () => { }}
+            hitFunc={() => { }}
             {...highlightedRef.current.highlight}
             scaleX={1 / item.parent.stageScale}
             scaleY={1 / item.parent.stageScale}
@@ -711,7 +711,7 @@ const HtxBrushView = ({ item }) => {
         }}
       >
         <Group>
-          <LabelOnMask item={item} color={item.strokeColor}/>
+          <LabelOnMask item={item} color={item.strokeColor} />
         </Group>
       </Layer>
     </RegionWrapper>

src/stores/Annotation/Annotation.js

@@ -839,9 +839,12 @@ export const Annotation = types
       };
 
 
-      //TODO: MST is crashing if we don't validate areas?, this problem isn't happening locally. So to reproduce you have to test in production or environment
+      // TODO: MST is crashing if we don't validate areas?, this problem isn't
+      // happening locally. So to reproduce you have to test in production or environment
       const area = self?.areas?.put(areaRaw);
 
+      object?.afterResultCreated?.(area);
+
       if (!area) return;
 
       if (!area.classification) getEnv(self).events.invoke('entityCreate', area);

src/tags/object/Image/Image.js

@@ -515,6 +515,13 @@ const Model = types.model({
 
       createImageEntities();
     }
+  
+    function afterResultCreated(region) {
+      if (!region) return;
+      if (!self.multiImage) return;
+
+      region.setItemIndex?.(self.currentImage);
+    }
 
     function getToolsManager() {
       return manager;
@@ -523,6 +530,7 @@ const Model = types.model({
     return {
       afterAttach,
       getToolsManager,
+      afterResultCreated,
     };
   }).extend((self) => {
     let skipInteractions = false;
@@ -631,6 +639,7 @@ const Model = types.model({
     setCurrentImage(index = 0) {
       index = index ?? 0;
       if (index === self.currentImage) return;
+
       self.currentImage = index;
       self.currentImageEntity = self.findImageEntity(index);
       if (isFF(FF_LSDV_4583_6)) self.preloadImages();

src/tools/Brush.js

@@ -182,9 +182,15 @@ const _Tool = types
         )
           return;
         const c = self.control;
+        const o = self.obj;
 
-        // Reset the timer if a user started drawing again
         brush = self.getSelectedShape;
+
+        // prevent drawing when current image is
+        // different from image where the brush was started
+        if (o && brush && o.multiImage && o.currentImage !== brush.item_index) return;
+
+        // Reset the timer if a user started drawing again
         if (brush && brush.type === 'brushregion') {
           self.annotation.history.freeze();
           self.mode = 'drawing';