fix(core): add support for reading and writing ISO-8859-1/latin1 files

Author: ithallys

packages/core/src/services/fileSystemService.ts

@@ -7,6 +7,7 @@
 import fs from 'node:fs/promises';
 import * as path from 'node:path';
 import { globSync } from 'glob';
+import { detectEncodingFromBuffer } from '../utils/systemEncoding.js';
 
 /**
  * Interface for file system operations that may be delegated to different implementations
@@ -47,7 +48,27 @@ export class StandardFileSystemService implements FileSystemService {
   }
 
   async writeTextFile(filePath: string, content: string): Promise<void> {
-    await fs.writeFile(filePath, content, 'utf-8');
+    let encoding: BufferEncoding = 'utf-8';
+    try {
+      const buffer = await fs.readFile(filePath);
+      const detected = detectEncodingFromBuffer(buffer);
+      if (detected) {
+        if (
+          detected.startsWith('iso-8859-') ||
+          detected.startsWith('windows-125')
+        ) {
+          encoding = 'latin1';
+        }
+      }
+    } catch (error) {
+      // If the file doesn't exist, we'll create it with the default UTF-8 encoding.
+      // For any other error (e.g., permissions), we should not silently ignore it,
+      // as that could lead to overwriting a file with the wrong encoding.
+      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
+        throw error;
+      }
+    }
+    await fs.writeFile(filePath, content, encoding);
   }
 
   findFiles(fileName: string, searchPaths: readonly string[]): string[] {

packages/core/src/utils/fileUtils.ts

@@ -15,6 +15,7 @@ import { ToolErrorType } from '../tools/tool-error.js';
 import { BINARY_EXTENSIONS } from './ignorePatterns.js';
 import { createRequire as createModuleRequire } from 'node:module';
 import { debugLogger } from './debugLogger.js';
+import { detectEncodingFromBuffer } from './systemEncoding.js';
 
 const requireModule = createModuleRequire(import.meta.url);
 
@@ -168,6 +169,21 @@ export async function readFileWithEncoding(filePath: string): Promise<string> {
 
   const bom = detectBOM(full);
   if (!bom) {
+    // No BOM. Try to detect encoding using chardet (via systemEncoding utility)
+    const detected = detectEncodingFromBuffer(full);
+    if (detected) {
+      // Node.js 'latin1' covers ISO-8859-1.
+      // We also map ISO-8859-2, windows-1252 etc. to latin1 as a best-effort fallback
+      // for single-byte encodings, since Node doesn't natively support them all
+      // and we want to preserve bytes as much as possible for simple text.
+      if (
+        detected.startsWith('iso-8859-') ||
+        detected.startsWith('windows-125')
+      ) {
+        return full.toString('latin1');
+      }
+    }
+
     // No BOM → treat as UTF‑8
     return full.toString('utf8');
   }

packages/core/src/utils/iso-encoding.test.ts

@@ -0,0 +1,77 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { readFileWithEncoding } from './fileUtils.js';
+import { StandardFileSystemService } from '../services/fileSystemService.js';
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+
+describe('ISO-8859-1 Encoding Support', () => {
+  let tmpDir: string;
+  let filePath: string;
+  const fileService = new StandardFileSystemService();
+
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gemini-iso-test-'));
+    filePath = path.join(tmpDir, 'test.txt');
+  });
+
+  afterEach(() => {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  it('should read ISO-8859-1 file correctly', async () => {
+    // "café na manhã" in ISO-8859-1
+    // c a f é _ n a _ m a n h ã
+    // 63 61 66 e9 20 6e 61 20 6d 61 6e 68 e3
+    const buffer = Buffer.from([
+      0x63, 0x61, 0x66, 0xe9, 0x20, 0x6e, 0x61, 0x20, 0x6d, 0x61, 0x6e, 0x68,
+      0xe3,
+    ]);
+    fs.writeFileSync(filePath, buffer);
+
+    const content = await readFileWithEncoding(filePath);
+    expect(content).toBe('café na manhã');
+  });
+
+  it('should preserve ISO-8859-1 encoding when writing', async () => {
+    // "café na manhã"
+    const buffer = Buffer.from([
+      0x63, 0x61, 0x66, 0xe9, 0x20, 0x6e, 0x61, 0x20, 0x6d, 0x61, 0x6e, 0x68,
+      0xe3,
+    ]);
+    fs.writeFileSync(filePath, buffer);
+
+    // Verify initial read
+    const content = await readFileWithEncoding(filePath);
+    expect(content).toBe('café na manhã');
+
+    // Write new content "café na manhã updated"
+    await fileService.writeTextFile(filePath, 'café na manhã updated');
+
+    // Read back as buffer to check encoding
+    const newBuffer = fs.readFileSync(filePath);
+
+    // Expect "café na manhã updated" in ISO-8859-1
+    // Original: 13 bytes. " updated": 8 bytes. Total 21 bytes.
+    expect(newBuffer.length).toBe(21);
+
+    const str = newBuffer.toString('latin1');
+    expect(str).toBe('café na manhã updated');
+  });
+
+  it('should fallback to UTF-8 for new files', async () => {
+    const newFilePath = path.join(tmpDir, 'new.txt');
+    await fileService.writeTextFile(newFilePath, 'café');
+
+    const buffer = fs.readFileSync(newFilePath);
+    // UTF-8 for é is C3 A9. Total 5 bytes.
+    expect(buffer.length).toBe(5);
+    expect(buffer.includes(0xc3)).toBe(true);
+  });
+});