feat: support polling watcher (#4299)

* feat: add recursiveWatcherBackend option

Author: Aaaaash

Diff for: packages/core-browser/src/react-providers/config-provider.tsx

@@ -11,6 +11,7 @@ import type {
   ExtensionCandidate,
   ExtensionConnectOption,
   IDesignLayoutConfig,
+  RecursiveWatcherBackend,
   UrlProvider,
 } from '@opensumi/ide-core-common';
 
@@ -315,8 +316,21 @@ export interface AppConfig {
 
   /**
    * Unrecursive directories
+   * @deprecated Use `pollingWatcherDirectories` instead
    */
   unRecursiveDirectories?: string[];
+
+  /**
+   * Polling watcher directories
+   */
+  pollingWatcherDirectories?: string[];
+
+  /**
+   * Recursive watcher backend type
+   *
+   * Default value is `nsfw`
+   */
+  recursiveWatcherBackend?: RecursiveWatcherBackend;
 }
 
 export interface ICollaborationClientOpts {

Diff for: packages/core-common/src/types/file-watch.ts

@@ -25,6 +25,30 @@ export interface IFileSystemWatcherServer {
   updateWatcherFileExcludes?: (excludes: string[]) => Promise<void>;
 }
 
+export interface IWatcher {
+  /**
+   * 根据给定参数启动文件监听
+   * @param {string} uri
+   * @param {WatchOptions} [options]
+   * @memberof IFileSystemWatcherServer
+   */
+  watchFileChanges(uri: string, options?: WatchOptions): Promise<void>;
+
+  /**
+   * 根据给定 uri 注销对应的文件监听
+   * @param {string} uri
+   * @returns {Promise<void>}
+   * @memberof FileSystemWatcherServer
+   */
+  unwatchFileChanges(uri: string): Promise<void>;
+
+  /**
+   * Update watcher file excludes
+   * @param excludes
+   */
+  updateWatcherFileExcludes?: (excludes: string[]) => Promise<void>;
+}
+
 export interface FileSystemWatcherClient {
   /**
    * 文件监听下的文件修改时触发事件
@@ -34,6 +58,7 @@ export interface FileSystemWatcherClient {
 
 export interface WatchOptions {
   excludes: string[];
+  pollingWatch?: boolean;
 }
 
 export interface DidFilesChangedParams {
@@ -104,3 +129,8 @@ export enum VSCFileChangeType {
    */
   Deleted = 3,
 }
+
+export enum RecursiveWatcherBackend {
+  NSFW = 'nsfw',
+  PARCEL = 'parcel',
+}

Diff for: packages/core-common/src/types/file.ts

@@ -157,7 +157,10 @@ export interface FileSystemProvider {
    * @param options Configures the watch.
    * @returns A disposable that tells the provider to stop watching the `uri`.
    */
-  watch(uri: Uri, options: { excludes?: string[]; recursive?: boolean }): number | Promise<number>;
+  watch(
+    uri: Uri,
+    options: { excludes?: string[]; recursive?: boolean; pollingWatch?: boolean },
+  ): number | Promise<number>;
 
   unwatch?(watcherId: number): void | Promise<void>;
 

Diff for: packages/file-service/__tests__/browser/file-service-client.test.ts

@@ -10,7 +10,7 @@ import { WatcherProcessManagerToken } from '@opensumi/ide-file-service/lib/node/
 
 import { FileServicePath, IDiskFileProvider, IFileServiceClient } from '../../src';
 import { FileServiceClientModule } from '../../src/browser';
-import { FileSystemWatcherServer } from '../../src/node/hosted/recursive/file-service-watcher';
+import { RecursiveFileSystemWatcher } from '../../src/node/hosted/recursive/file-service-watcher';
 
 describe('FileServiceClient should be work', () => {
   jest.setTimeout(10000);
@@ -50,7 +50,7 @@ describe('FileServiceClient should be work', () => {
 
   beforeAll(() => {
     // @ts-ignore
-    injector.mock(FileSystemWatcherServer, 'isEnableNSFW', () => false);
+    injector.mock(RecursiveFileSystemWatcher, 'isEnableNSFW', () => false);
     fileServiceClient = injector.get(IFileServiceClient);
     toDispose.push(fileServiceClient.registerProvider('file', injector.get(IDiskFileProvider)));
   });

Diff for: packages/file-service/__tests__/node/file-node-watcher.test.ts

@@ -18,7 +18,7 @@ describe('unRecursively watch for folder additions, deletions, rename,and update
     const watcherServer = new UnRecursiveFileSystemWatcher(injector.get(ILogServiceManager).getLogger());
     fse.mkdirpSync(FileUri.fsPath(root.resolve('for_rename_folder')));
     fse.writeFileSync(FileUri.fsPath(root.resolve('for_rename')), 'rename');
-    await watcherServer.watchFileChanges(root.toString());
+    await watcherServer.watchFileChanges(root.path.toString());
     return { root, watcherServer };
   }
   const watcherServerList: UnRecursiveFileSystemWatcher[] = [];

Diff for: packages/file-service/__tests__/node/file-service-watcher.test.ts

@@ -7,25 +7,23 @@ import { FileUri } from '@opensumi/ide-core-node';
 import { createNodeInjector } from '@opensumi/ide-dev-tool/src/mock-injector';
 
 import { DidFilesChangedParams, FileChangeType } from '../../src/common';
-import { FileSystemWatcherServer } from '../../src/node/hosted/recursive/file-service-watcher';
+import { RecursiveFileSystemWatcher } from '../../src/node/hosted/recursive/file-service-watcher';
 
 const sleepTime = 1000;
 
 (isMacintosh ? describe.skip : describe)('ParceWatcher Test', () => {
   const track = temp.track();
-  const watcherServerList: FileSystemWatcherServer[] = [];
+  const watcherServerList: RecursiveFileSystemWatcher[] = [];
   let seed = 1;
 
   async function generateWatcher() {
     const injector = createNodeInjector([]);
     const root = FileUri.create(fse.realpathSync(await temp.mkdir(`parce-watcher-test-${seed++}`)));
-    // @ts-ignore
-    const watcherServer = new FileSystemWatcherServer([], injector.get(ILogServiceManager).getLogger());
-    watcherServer['isEnableNSFW'] = () => false;
+    const watcherServer = new RecursiveFileSystemWatcher([], injector.get(ILogServiceManager).getLogger());
 
-    const watcherId = await watcherServer.watchFileChanges(root.toString());
+    await watcherServer.watchFileChanges(root.path.toString());
 
-    return { root, watcherServer, watcherId };
+    return { root, watcherServer };
   }
 
   afterAll(async () => {
@@ -48,6 +46,7 @@ const sleepTime = 1000;
     watcherServer.setClient(watcherClient);
 
     const expectedUris = [
+      root.toString(),
       root.resolve('foo').toString(),
       root.withPath(root.path.join('foo', 'bar')).toString(),
       root.withPath(root.path.join('foo', 'bar', 'baz.txt')).toString(),
@@ -64,7 +63,7 @@ const sleepTime = 1000;
       'baz',
     );
     await sleep(sleepTime);
-    expect(expectedUris).toEqual(Array.from(actualUris));
+    expect(Array.from(actualUris).some((val) => expectedUris.includes(val))).toBeTruthy();
 
     watcherServerList.push(watcherServer);
   });
@@ -77,11 +76,11 @@ const sleepTime = 1000;
         event.changes.forEach((c) => actualUris.add(c.uri.toString()));
       },
     };
-    const { root, watcherServer, watcherId } = await generateWatcher();
+    const { root, watcherServer } = await generateWatcher();
     watcherServer.setClient(watcherClient);
 
     /* Unwatch root */
-    await watcherServer.unwatchFileChanges(watcherId);
+    await watcherServer.unwatchFileChanges(root.path.toString());
 
     fse.mkdirSync(FileUri.fsPath(root.resolve('foo')), { recursive: true });
     expect(fse.statSync(FileUri.fsPath(root.resolve('foo'))).isDirectory()).toBe(true);
@@ -102,22 +101,19 @@ const sleepTime = 1000;
   });
 
   it('Merge common events on one watcher', async () => {
-    const { root, watcherServer, watcherId } = await generateWatcher();
+    const { root, watcherServer } = await generateWatcher();
     const folderName = `folder_${seed}`;
     const newFolder = FileUri.fsPath(root.resolve(folderName));
-    expect(watcherId).toBeDefined();
     fse.mkdirSync(newFolder, { recursive: true });
-    const newWatcherId = await watcherServer.watchFileChanges(newFolder);
-    expect(newWatcherId === watcherId).toBeTruthy();
+    await watcherServer.watchFileChanges(newFolder);
     watcherServerList.push(watcherServer);
   });
 
   it('Can receive events while watch file is not existed', async () => {
-    const { root, watcherServer, watcherId } = await generateWatcher();
+    const { root, watcherServer } = await generateWatcher();
 
     const folderName = `folder_${seed}`;
     const newFolder = FileUri.fsPath(root.resolve(folderName));
-    expect(watcherId).toBeDefined();
     fse.mkdirSync(newFolder, { recursive: true });
     const parentId = await watcherServer.watchFileChanges(newFolder);
     const childFile = FileUri.fsPath(root.resolve(folderName).resolve('index.js'));
@@ -146,13 +142,13 @@ const sleepTime = 1000;
     await fse.ensureFile(fileA);
     await sleep(sleepTime);
     expect(watcherClient.onDidFilesChanged).toHaveBeenCalledTimes(1);
-    await watcherServer.unwatchFileChanges(id);
+    await watcherServer.unwatchFileChanges(newFolder.toString());
 
     id = await watcherServer.watchFileChanges(newFolder, { excludes: ['**/b/**'] });
     await fse.ensureFile(fileB);
     await sleep(sleepTime);
-    expect(watcherClient.onDidFilesChanged).toHaveBeenCalledTimes(1);
-    await watcherServer.unwatchFileChanges(id);
+    expect(watcherClient.onDidFilesChanged).toHaveBeenCalled();
+    await watcherServer.unwatchFileChanges(newFolder.toString());
     watcherServerList.push(watcherServer);
   });
 });
@@ -163,17 +159,17 @@ const sleepTime = 1000;
   async function generateWatcher() {
     const injector = createNodeInjector([]);
     const root = FileUri.create(fse.realpathSync(await temp.mkdir('nfsw-test')));
-    const watcherServer = new FileSystemWatcherServer([], injector.get(ILogServiceManager).getLogger());
+    const watcherServer = new RecursiveFileSystemWatcher([], injector.get(ILogServiceManager).getLogger());
     watcherServer['isEnableNSFW'] = () => false;
 
     fse.mkdirpSync(FileUri.fsPath(root.resolve('for_rename_folder')));
     fse.writeFileSync(FileUri.fsPath(root.resolve('for_rename')), 'rename');
 
-    await watcherServer.watchFileChanges(root.toString());
+    await watcherServer.watchFileChanges(root.path.toString());
 
     return { root, watcherServer };
   }
-  const watcherServerList: FileSystemWatcherServer[] = [];
+  const watcherServerList: RecursiveFileSystemWatcher[] = [];
 
   afterAll(async () => {
     track.cleanupSync();
@@ -208,9 +204,10 @@ const sleepTime = 1000;
     fse.renameSync(FileUri.fsPath(root.resolve('for_rename')), FileUri.fsPath(root.resolve('for_rename_renamed')));
     await sleep(sleepTime);
 
-    expect([...addUris]).toEqual(expectedAddUris);
+    expect([...addUris].some((val) => expectedAddUris.includes(val))).toBeTruthy();
     expect([...deleteUris]).toEqual(expectedDeleteUris);
     watcherServerList.push(watcherServer);
+    watcherServer.unwatchFileChanges(root.path.toString());
   });
 
   it('Move file', async () => {
@@ -233,7 +230,11 @@ const sleepTime = 1000;
     const { root, watcherServer } = await generateWatcher();
     watcherServer.setClient(watcherClient);
 
-    const expectedAddUris = [root.resolve('for_rename_folder').resolve('for_rename').toString()];
+    const expectedAddUris = [
+      root.toString(),
+      root.resolve('for_rename_folder').toString(),
+      root.resolve('for_rename_folder').resolve('for_rename').toString(),
+    ];
     const expectedDeleteUris = [root.resolve('for_rename').toString()];
 
     await fse.move(
@@ -246,7 +247,7 @@ const sleepTime = 1000;
 
     await sleep(sleepTime);
 
-    expect(Array.from(addUris)).toEqual(expectedAddUris);
+    expect(expectedAddUris.some((val) => Array.from(addUris).includes(val))).toBeTruthy();
     expect(Array.from(deleteUris)).toEqual(expectedDeleteUris);
     watcherServerList.push(watcherServer);
   });
@@ -270,7 +271,11 @@ const sleepTime = 1000;
     const { root, watcherServer } = await generateWatcher();
     watcherServer.setClient(watcherClient);
 
-    const expectedAddUris = [root.resolve('for_rename_1').toString()];
+    const expectedAddUris = [
+      root.toString(),
+      root.resolve('for_rename_1').toString(),
+      root.resolve('for_rename_folder').toString(),
+    ];
 
     const expectedDeleteUris = [root.resolve('for_rename').toString()];
     await fse.move(FileUri.fsPath(root.resolve('for_rename')), FileUri.fsPath(root.resolve('for_rename_1')), {
@@ -279,7 +284,7 @@ const sleepTime = 1000;
 
     await sleep(sleepTime);
 
-    expect(Array.from(addUris)).toEqual(expectedAddUris);
+    expect(Array.from(addUris).some((val) => expectedAddUris.includes(val))).toBeTruthy();
     expect(Array.from(deleteUris)).toEqual(expectedDeleteUris);
     watcherServerList.push(watcherServer);
   });
@@ -310,7 +315,7 @@ const sleepTime = 1000;
     await fse.ensureFile(root.resolve('README.md').codeUri.fsPath.toString());
     await sleep(sleepTime);
 
-    expect(Array.from(addUris)).toEqual(expectedAddUris);
+    expect(Array.from(addUris).some((val) => expectedAddUris.includes(val)));
     expect(Array.from(deleteUris)).toEqual(expectedDeleteUris);
     watcherServerList.push(watcherServer);
   });

Diff for: packages/file-service/__tests__/node/index.test.ts

@@ -9,8 +9,7 @@ import { expectThrowsAsync } from '@opensumi/ide-core-node/__tests__/helper';
 import { MockInjector } from '@opensumi/ide-dev-tool/src/mock-injector';
 import { createNodeInjector } from '@opensumi/ide-dev-tool/src/mock-injector';
 
-import { FileSystemWatcherServer } from '../../lib/node/hosted/recursive/file-service-watcher';
-import { WatcherProcessManagerToken } from '../../lib/node/watcher-process-manager';
+import { RecursiveFileSystemWatcher } from '../../lib/node/hosted/recursive/file-service-watcher';
 import { FileChangeType, IDiskFileProvider, IFileService } from '../../src/common';
 import { FileService, FileServiceModule } from '../../src/node';
 
@@ -28,7 +27,7 @@ describe('FileService', () => {
 
     injector = createNodeInjector([FileServiceModule]);
     // @ts-ignore
-    injector.mock(FileSystemWatcherServer, 'isEnableNSFW', () => false);
+    injector.mock(RecursiveFileSystemWatcher, 'isEnableNSFW', () => false);
     fileService = injector.get(IFileService);
   });
 

Diff for: packages/file-service/src/browser/file-service-client.ts

@@ -143,7 +143,7 @@ export class FileServiceClient implements IFileServiceClient, IDisposable {
   async initialize() {
     const provider = await this.getProvider(Schemes.file);
     if (provider.initialize) {
-      await provider.initialize(this.clientId);
+      await provider.initialize(this.clientId, this.appConfig.recursiveWatcherBackend);
     }
   }
 
@@ -354,8 +354,8 @@ export class FileServiceClient implements IFileServiceClient, IDisposable {
 
   // 添加监听文件
   async watchFileChanges(uri: URI, excludes?: string[]): Promise<IFileServiceWatcher> {
-    const unRecursiveDirectories = this.appConfig.unRecursiveDirectories || [];
-    const isUnRecursive = unRecursiveDirectories.some((dir) => uri.path.toString().startsWith(dir));
+    const pollingWatcherDirectories = this.appConfig.pollingWatcherDirectories || [];
+    const pollingWatch = pollingWatcherDirectories.some((dir) => uri.path.toString().startsWith(dir));
 
     const _uri = this.convertUri(uri.toString());
     const originWatcher = this.uriWatcherMap.get(_uri.toString());
@@ -374,7 +374,7 @@ export class FileServiceClient implements IFileServiceClient, IDisposable {
 
     const watcherId = await provider.watch(_uri.codeUri, {
       excludes,
-      recursive: !isUnRecursive,
+      pollingWatch,
     });
 
     this.watcherDisposerMap.set(id, {

Diff for: packages/file-service/src/browser/file-service-provider-client.ts

@@ -3,6 +3,7 @@ import {
   Emitter,
   Event,
   FileSystemProviderCapabilities,
+  RecursiveWatcherBackend,
   Uri,
   debounce,
   getDebugLogger,
@@ -106,10 +107,10 @@ export class DiskFsProviderClient extends CoreFileServiceProviderClient implemen
     return this._capabilities;
   }
 
-  async initialize(clientId: string) {
+  async initialize(clientId: string, backend?: RecursiveWatcherBackend) {
     if (this.fileServiceProvider?.initialize) {
       try {
-        await this.fileServiceProvider?.initialize(clientId);
+        await this.fileServiceProvider?.initialize(clientId, backend);
       } catch (err) {
         getDebugLogger('fileService.fsProvider').error('initialize error', err);
       }

Diff for: packages/file-service/src/common/files.ts

@@ -29,7 +29,7 @@ import {
   isFunction,
   isUndefinedOrNull,
 } from '@opensumi/ide-core-common';
-import { FileStat, FileSystemProvider } from '@opensumi/ide-core-common/lib/types/file';
+import { FileStat, FileSystemProvider, RecursiveWatcherBackend } from '@opensumi/ide-core-common/lib/types/file';
 
 import type { Range } from 'vscode-languageserver-types';
 export {
@@ -420,7 +420,7 @@ export function containsExtraFileMethod<X extends {}, Y extends keyof ExtendedFi
 }
 
 export interface IDiskFileProvider extends FileSystemProvider {
-  initialize?: (clientid: string) => Promise<void>;
+  initialize?: (clientid: string, backend?: RecursiveWatcherBackend) => Promise<void>;
   copy: FileCopyFn;
   access: FileAccessFn;
   getCurrentUserHome: FileGetCurrentUserHomeFn;