improve help

Author: DenisPalnitsky

README.md

@@ -1,41 +1,57 @@
-# File-for-AI
+# File-for-AI 
 
 File-for-AI is an open-source tool designed to compile all text files in a directory into a single file. This is particularly useful for preparing data to be added to AI models or chat GPT.
 
 ## Description
 
 This tool traverses through a specified directory, reads all the text files (while ignoring non-text files and files specified in .gitignore), and compiles them into a single output file. This output file can then be used for various purposes such as training AI models or chat GPT.
 
-
 ## Usage
 
-Download the latest binary for the os of your coice from the [releases page](https://github.com/num30/file-for-ai/releases) it's under `Assets` section.
+Download the latest binary for the os of your coice from the [releases page](https://github.com/num30/file-for-ai/releases) under the `Assets` section.
+
+To use this tool, you need to provide at least a directory path or a glob pattern as an argument. If an output file name is not provided, it will default to "file-for-ai.txt."
 
-To use this tool, you need to provide at least a directory path as an argument. If an output file name is not provided, it will default to "file-for-ai.txt".
 
-```bash
-file-for-ai <directory> 
+``` bash
+file-for-ai <directory|pattern> [--output file]
 ```
 
 For example:
 
-```bash
+``` bash
 file-for-ai /path/to/your/directory
 ```
 
 This will create a file named "file-for-ai.txt" in your current directory with the contents of all text files in the specified directory.
 
+You can also use a glob pattern:
+
+``` bash
+file-for-ai *.txt
+```
+
+This will create a file named "file-for-ai.txt" in your current directory with the contents of all text files matching the pattern.
+
+To specify a custom output file name:
+
+``bash
+file-for-ai /path/to/your/directory --output custom-output.txt
+````
+
+This will create a file named "custom-output.txt" in your current directory with the contents of all text files in the specified directory.
+
 ## Flags
+
 The tool accepts the following flags:
 
-- `--model` or `-m`: Specifies the model to use for toke counting. It should be one of the available models in [tiktoken-go](https://github.com/pkoukk/tiktoken-go?tab=readme-ov-file#available-encodings). Default is "gpt-4".
-- `--output` or `-o`: Specifies the name of the output file. Default is "file-for-ai.txt".
+- `--model`: Specifies the model to use for token counting. It should be one of the available models in [tiktoken-go](https://github.com/pkoukk/tiktoken-go?tab=readme-ov-file#available-encodings). Default is "gpt-4".
+- `--output`: Specifies the name of the output file. Default is "file-for-ai.txt."
+- `--ignore-gitignore`: If set to true, the tool will ignore the .gitignore file. Default is false.
+- `--process-non-text`: If set to true, the tool will process non-text files. Default is false.
 
 For example:
 
-``` bash
-file-for-ai /path/to/your/directory --model gpt-4o --output my-output.txt
-```
-
-
-This will create a file named "my-output.txt" in your current directory with the contents of all text files in the specified directory, token counting using the "gpt-4" model.
+``bash
+file-for-ai /path/to/your/directory --model gpt-4 --output my-output.txt
+```

main.go

@@ -18,19 +18,25 @@ type Config struct {
 	Model           string `flag:"model" envvar:"MODEL" default:"gpt-4"`
 	OutputFile      string `flag:"output" envvar:"FILE" default:"file-for-ai.txt"`
 	IgnoreGitIgnore bool   `flag:"ignore-gitignore" envvar:"IGNORE_GITIGNORE" default:"false"`
+	ProcessNonText  bool   `flag:"process-non-text" envvar:"PROCESS_NON_TEXT" default:"false"`
 }
 
+var conf Config
+
 func main() {
 
-	var conf Config
 	err := config.NewConfReader("file-for-ai").Read(&conf)
 	if err != nil {
 		panic(err)
 	}
 
 	if len(os.Args) < 2 {
 		fmt.Println("Error: Directory path or glob pattern is required.")
-		fmt.Println("Usage: file-for-ai <directory|pattern> [output file]")
+		fmt.Println("Usage: file-for-ai <directory|pattern> --output [output file]")
+		fmt.Println("\nExamples:")
+		fmt.Println("  file-for-ai /path/to/directory")
+		fmt.Println("  file-for-ai '*.txt'")
+		fmt.Println("  file-for-ai /path/to/directory --output custom-output.txt")
 		os.Exit(1)
 	}
 
@@ -126,7 +132,7 @@ func processFile(basePath, path string, info os.FileInfo, outputFile *os.File, t
 		return err
 	}
 
-	if !info.IsDir() && !isGitIgnored(relativePath, info.IsDir()) && isTextFile(path) && !strings.HasPrefix(path, ".") {
+	if !info.IsDir() && !isGitIgnored(relativePath, info.IsDir()) && extensionBasedFilter(path) && !strings.HasPrefix(path, ".") {
 		fileContents, err := os.ReadFile(path)
 		if err != nil {
 			fmt.Println("Error reading file:", path, err)
@@ -178,7 +184,11 @@ func isGitIgnored(path string, isDir bool) bool {
 	return false
 }
 
-func isTextFile(path string) bool {
+func extensionBasedFilter(path string) bool {
+	if conf.ProcessNonText {
+		return true
+	}
+
 	ext := strings.ToLower(filepath.Ext(path))
 	return !nonTextFileExtensions[ext]
 }