feat: add option to pass skip special tokens flag to decode (#162)

## Description

This PR adds skipSpecialTokens flag to argument list of decode method

### Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] Documentation update (improves or adds clarity to existing
documentation)

### Tested on

- [x] iOS
- [x] Android

### Testing instructions

<!-- Provide step-by-step instructions on how to test your changes.
Include setup details if necessary. -->

### Screenshots

<!-- Add screenshots here, if applicable -->

### Related issues

<!-- Link related issues here using #issue-number -->

### Checklist

- [x] I have performed a self-review of my code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have updated the documentation accordingly
- [x] My changes generate no new warnings

### Additional notes

<!-- Include any additional information, assumptions, or context that
reviewers might need to understand this PR. -->

Author: NorbertKlockiewicz

android/libs/executorch.aar

@@ -32,10 +32,11 @@ class Tokenizer(
 
   override fun decode(
     input: ReadableArray,
+    skipSpecialTokens: Boolean,
     promise: Promise,
   ) {
     try {
-      promise.resolve(tokenizer.decode(createIntArray(input)))
+      promise.resolve(tokenizer.decode(createIntArray(input), skipSpecialTokens))
     } catch (e: Exception) {
       promise.reject(e.message!!, ETError.UndefinedError.toString())
     }

android/src/main/java/com/swmansion/rnexecutorch/Tokenizer.kt

@@ -5,6 +5,8 @@
 - (instancetype)initWithTokenizerPath:(NSString *)tokenizerPath;
 - (NSArray<NSNumber *> *)encode:(NSString *)text;
 - (NSString *)decode:(NSArray<NSNumber *> *)tokenIds;
+- (NSString *)decode:(NSArray<NSNumber *> *)tokenIds
+    skipSpecialTokens:(BOOL)skipSpecialTokens;
 - (NSUInteger)getVocabSize;
 - (NSString *)idToToken:(NSInteger)tokenId;
 - (NSInteger)tokenToId:(NSString *)token;

ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/ExecutorchLib

@@ -10,7 +10,7 @@
 		</data>
 		<key>Headers/HuggingFaceTokenizer.h</key>
 		<data>
-		cVZsliuTmV2umgK601d6PamTRSw=
+		OJmjnUO2q4YZvXqqz4vCymvZFP0=
 		</data>
 		<key>Headers/LLaMARunner.h</key>
 		<data>
@@ -34,7 +34,7 @@
 		<dict>
 			<key>hash2</key>
 			<data>
-			0ETM5qw12+W8ULx2zP2UkVomFrBRnwAr8I7po2ACk/k=
+			KIuu4Sb2dgJ9swgF7bGppf5T9KjRhVQ6ERGHAvs4D/Q=
 			</data>
 		</dict>
 		<key>Headers/LLaMARunner.h</key>

ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/Headers/HuggingFaceTokenizer.h

@@ -5,6 +5,8 @@
 - (instancetype)initWithTokenizerPath:(NSString *)tokenizerPath;
 - (NSArray<NSNumber *> *)encode:(NSString *)text;
 - (NSString *)decode:(NSArray<NSNumber *> *)tokenIds;
+- (NSString *)decode:(NSArray<NSNumber *> *)tokenIds
+    skipSpecialTokens:(BOOL)skipSpecialTokens;
 - (NSUInteger)getVocabSize;
 - (NSString *)idToToken:(NSInteger)tokenId;
 - (NSInteger)tokenToId:(NSString *)token;

ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/_CodeSignature/CodeResources

@@ -10,7 +10,7 @@
 		</data>
 		<key>Headers/HuggingFaceTokenizer.h</key>
 		<data>
-		cVZsliuTmV2umgK601d6PamTRSw=
+		OJmjnUO2q4YZvXqqz4vCymvZFP0=
 		</data>
 		<key>Headers/LLaMARunner.h</key>
 		<data>
@@ -34,7 +34,7 @@
 		<dict>
 			<key>hash2</key>
 			<data>
-			0ETM5qw12+W8ULx2zP2UkVomFrBRnwAr8I7po2ACk/k=
+			KIuu4Sb2dgJ9swgF7bGppf5T9KjRhVQ6ERGHAvs4D/Q=
 			</data>
 		</dict>
 		<key>Headers/LLaMARunner.h</key>

ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/ExecutorchLib

@@ -32,10 +32,11 @@ - (void)encode:(NSString *)input
 }
 
 - (void)decode:(NSArray *)input
-       resolve:(RCTPromiseResolveBlock)resolve
-        reject:(RCTPromiseRejectBlock)reject {
+    skipSpecialTokens:(BOOL)skipSpecialTokens
+              resolve:(RCTPromiseResolveBlock)resolve
+               reject:(RCTPromiseRejectBlock)reject {
   @try {
-    resolve([tokenizer decode:input]);
+    resolve([tokenizer decode:input skipSpecialTokens:skipSpecialTokens]);
   } @catch (NSException *exception) {
     reject(@"tokenizer_error",
            [NSString stringWithFormat:@"%@", exception.reason], nil);

ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/Headers/HuggingFaceTokenizer.h

@@ -3,7 +3,7 @@ import { TurboModuleRegistry } from 'react-native';
 
 export interface Spec extends TurboModule {
   load(tokenizerSource: string): Promise<number>;
-  decode(input: number[]): Promise<string>;
+  decode(input: number[], skipSpecialTokens: boolean): Promise<string>;
   encode(input: string): Promise<number[]>;
   getVocabSize(): Promise<number>;
   idToToken(tokenId: number): Promise<string>;

ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/_CodeSignature/CodeResources

@@ -5,6 +5,8 @@
 - (instancetype)initWithTokenizerPath:(NSString *)tokenizerPath;
 - (NSArray<NSNumber *> *)encode:(NSString *)text;
 - (NSString *)decode:(NSArray<NSNumber *> *)tokenIds;
+- (NSString *)decode:(NSArray<NSNumber *> *)tokenIds
+    skipSpecialTokens:(BOOL)skipSpecialTokens;
 - (NSUInteger)getVocabSize;
 - (NSString *)idToToken:(NSInteger)tokenId;
 - (NSInteger)tokenToId:(NSString *)token;

ios/RnExecutorch/Tokenizer.mm

@@ -44,12 +44,18 @@ - (instancetype)initWithTokenizerPath:(NSString *)tokenizerPath {
 }
 
 - (NSString *)decode:(NSArray<NSNumber *> *)tokenIds {
+  return [self decode:tokenIds skipSpecialTokens:NO];
+}
+
+- (NSString *)decode:(NSArray<NSNumber *> *)tokenIds
+    skipSpecialTokens:(BOOL)skipSpecialTokens {
   std::vector<int32_t> stdTokenIds;
   stdTokenIds.reserve([tokenIds count]);
   for (NSNumber *tokenId in tokenIds) {
     stdTokenIds.push_back([tokenId intValue]);
   }
-  std::string decodedString = _tokenizer->Decode(stdTokenIds);
+  std::string decodedString =
+      _tokenizer->Decode(stdTokenIds, skipSpecialTokens);
   return [NSString stringWithUTF8String:decodedString.c_str()];
 }