Skip to content

Commit 76e35fd

Browse files
Eric-Ansdevmobile
Eric-Ans
and
devmobile
authored
feat: Added cornerPoints to Block, Line, Element in TextDetectionResult (#12)
* Add additional information to results * ios update * remove confidence and angle, not available on ios * update mlkitTextRecognitionVersion to 19.0.1 --------- Co-authored-by: devmobile <[email protected]>
1 parent 476253f commit 76e35fd

File tree

8 files changed

+2501
-835
lines changed

8 files changed

+2501
-835
lines changed

‎android/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ ext {
33
androidxAppCompatVersion = project.hasProperty('androidxAppCompatVersion') ? rootProject.ext.androidxAppCompatVersion : '1.6.1'
44
androidxJunitVersion = project.hasProperty('androidxJunitVersion') ? rootProject.ext.androidxJunitVersion : '1.1.5'
55
androidxEspressoCoreVersion = project.hasProperty('androidxEspressoCoreVersion') ? rootProject.ext.androidxEspressoCoreVersion : '3.5.1'
6-
mlkitTextRecognitionVersion = project.hasProperty('mlkitTextRecognitionVersion') ? rootProject.ext.mlkitTextRecognitionVersion : '18.0.2'
6+
mlkitTextRecognitionVersion = project.hasProperty('mlkitTextRecognitionVersion') ? rootProject.ext.mlkitTextRecognitionVersion : '19.0.1'
77
}
88

99
buildscript {

‎android/src/main/java/com/pantrist/ml/CapacitorPluginMlKitTextRecognition.kt

+92-69
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import com.getcapacitor.*
55
import com.getcapacitor.annotation.CapacitorPlugin
66
import java.io.IOException
77
import android.graphics.BitmapFactory
8+
import android.graphics.Point
89
import android.graphics.Rect
910

1011
import com.google.mlkit.vision.common.InputImage
@@ -13,83 +14,105 @@ import com.google.mlkit.vision.text.latin.TextRecognizerOptions
1314

1415
@CapacitorPlugin
1516
class CapacitorPluginMlKitTextRecognition : Plugin() {
16-
@PluginMethod
17-
fun detectText(call: PluginCall) {
18-
val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS)
17+
@PluginMethod
18+
fun detectText(call: PluginCall) {
19+
val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS)
1920

20-
val encodedImage = call.getString("base64Image")
21-
if (encodedImage == null) {
22-
call.reject("No image is given!")
23-
return
24-
}
25-
val rotation = call.getInt("rotation") ?: 0
26-
27-
val image: InputImage
28-
try {
29-
val decodedString: ByteArray = Base64.decode(encodedImage, Base64.DEFAULT);
30-
val decodedByte = BitmapFactory.decodeByteArray(decodedString, 0, decodedString.size)
31-
if (decodedByte == null) {
32-
call.reject("Decoded image is null")
33-
return
34-
}
35-
image = InputImage.fromBitmap(decodedByte, rotation)
36-
} catch (e: IOException) {
37-
call.reject("Unable to parse image")
38-
return
39-
}
21+
val encodedImage = call.getString("base64Image")
22+
if (encodedImage == null) {
23+
call.reject("No image is given!")
24+
return
25+
}
26+
val rotation = call.getInt("rotation") ?: 0
4027

41-
recognizer.process(image)
42-
.addOnSuccessListener { visionText ->
43-
val ret = JSObject()
44-
ret.put("text", visionText.text)
28+
val image: InputImage
29+
try {
30+
val decodedString: ByteArray = Base64.decode(encodedImage, Base64.DEFAULT);
31+
val decodedByte = BitmapFactory.decodeByteArray(decodedString, 0, decodedString.size)
32+
if (decodedByte == null) {
33+
call.reject("Decoded image is null")
34+
return
35+
}
36+
image = InputImage.fromBitmap(decodedByte, rotation)
37+
} catch (e: IOException) {
38+
call.reject("Unable to parse image")
39+
return
40+
}
4541

46-
val textBlocks = JSArray()
47-
visionText.textBlocks.forEach { block ->
48-
val blockObject = JSObject();
49-
blockObject.put("text", block.text)
50-
blockObject.put("boundingBox", parseRectToJsObject(block.boundingBox))
51-
blockObject.put("recognizedLanguage", block.recognizedLanguage)
42+
recognizer.process(image)
43+
.addOnSuccessListener { visionText ->
44+
val ret = JSObject()
45+
ret.put("text", visionText.text)
5246

53-
val linesArray = JSArray()
54-
block.lines.forEach { line ->
55-
val lineObject = JSObject()
56-
lineObject.put("text", line.text)
57-
lineObject.put("boundingBox", parseRectToJsObject(line.boundingBox))
58-
lineObject.put("recognizedLanguage", line.recognizedLanguage)
47+
val textBlocks = JSArray()
48+
visionText.textBlocks.forEach { block ->
49+
val blockObject = JSObject();
50+
blockObject.put("text", block.text)
51+
blockObject.put("boundingBox", parseRectToJsObject(block.boundingBox))
52+
blockObject.put("recognizedLanguage", block.recognizedLanguage)
53+
blockObject.put("cornerPoints", parseCornerPointsToJsObject(block.cornerPoints))
5954

60-
val elementArray = JSArray()
61-
line.elements.forEach { element ->
62-
val elementObject = JSObject()
63-
elementObject.put("text", element.text)
64-
elementObject.put("boundingBox", parseRectToJsObject(element.boundingBox))
65-
elementObject.put("recognizedLanguage", line.recognizedLanguage)
66-
elementArray.put(elementObject)
67-
}
68-
lineObject.put("elements", elementArray)
69-
linesArray.put(lineObject)
70-
}
71-
blockObject.put("lines", linesArray)
72-
textBlocks.put(blockObject)
73-
};
74-
ret.put("blocks", textBlocks)
55+
val linesArray = JSArray()
56+
block.lines.forEach { line ->
57+
val lineObject = JSObject()
58+
lineObject.put("text", line.text)
59+
lineObject.put("boundingBox", parseRectToJsObject(line.boundingBox))
60+
lineObject.put("recognizedLanguage", line.recognizedLanguage)
61+
lineObject.put("cornerPoints", parseCornerPointsToJsObject(line.cornerPoints))
7562

76-
call.resolve(ret)
77-
}
78-
.addOnFailureListener { e ->
79-
call.reject("Unable process image!", e)
63+
val elementArray = JSArray()
64+
line.elements.forEach { element ->
65+
val elementObject = JSObject()
66+
elementObject.put("text", element.text)
67+
elementObject.put("boundingBox", parseRectToJsObject(element.boundingBox))
68+
elementObject.put("recognizedLanguage", line.recognizedLanguage)
69+
elementObject.put("cornerPoints", parseCornerPointsToJsObject(element.cornerPoints))
70+
elementArray.put(elementObject)
8071
}
72+
lineObject.put("elements", elementArray)
73+
linesArray.put(lineObject)
74+
}
75+
blockObject.put("lines", linesArray)
76+
textBlocks.put(blockObject)
77+
};
78+
ret.put("blocks", textBlocks)
79+
80+
call.resolve(ret)
81+
}
82+
.addOnFailureListener { e ->
83+
call.reject("Unable process image!", e)
84+
}
85+
}
86+
87+
private fun parseRectToJsObject(rect: Rect?): JSObject? {
88+
if (rect == null) {
89+
return null
8190
}
8291

83-
private fun parseRectToJsObject(rect: Rect?): JSObject? {
84-
if (rect == null) {
85-
return null
86-
}
92+
val returnObject = JSObject();
93+
returnObject.put("left", rect.left)
94+
returnObject.put("top", rect.top)
95+
returnObject.put("right", rect.right)
96+
returnObject.put("bottom", rect.bottom)
97+
return returnObject;
98+
}
8799

88-
val returnObject = JSObject();
89-
returnObject.put("left", rect.left)
90-
returnObject.put("top", rect.top)
91-
returnObject.put("right", rect.right)
92-
returnObject.put("bottom", rect.bottom)
93-
return returnObject;
100+
private fun parseCornerPointsToJsObject(cornerPoints: Array<Point>?): JSObject? {
101+
if (cornerPoints == null || cornerPoints.size != 4) {
102+
return null;
94103
}
95-
}
104+
val res = JSObject();
105+
res.put("topLeft", pointToJsObject(cornerPoints[0]));
106+
res.put("topRight", pointToJsObject(cornerPoints[1]));
107+
res.put("bottomRight", pointToJsObject(cornerPoints[2]));
108+
res.put("bottomLeft", pointToJsObject(cornerPoints[3]));
109+
return res;
110+
}
111+
112+
private fun pointToJsObject(pt: Point): JSObject {
113+
val res = JSObject();
114+
res.put("x", pt.x.toDouble());
115+
res.put("y", pt.y.toDouble());
116+
return res;
117+
}
118+
}

‎ios/Plugin.xcodeproj/project.pbxproj

-6
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
/* Begin PBXBuildFile section */
1010
03FC29A292ACC40490383A1F /* Pods_Plugin.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 3B2A61DA5A1F2DD4F959604D /* Pods_Plugin.framework */; };
1111
20C0B05DCFC8E3958A738AF2 /* Pods_PluginTests.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = F6753A823D3815DB436415E3 /* Pods_PluginTests.framework */; };
12-
2F98D68224C9AAE500613A4C /* CapacitorPluginMlKitTextRecognition.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2F98D68124C9AAE400613A4C /* CapacitorPluginMlKitTextRecognition.swift */; };
1312
50ADFF92201F53D600D50D53 /* Plugin.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 50ADFF88201F53D600D50D53 /* Plugin.framework */; };
1413
50ADFF97201F53D600D50D53 /* CapacitorPluginMlKitTextRecognitionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50ADFF96201F53D600D50D53 /* CapacitorPluginMlKitTextRecognitionTests.swift */; };
1514
50ADFF99201F53D600D50D53 /* CapacitorPluginMlKitTextRecognitionPlugin.h in Headers */ = {isa = PBXBuildFile; fileRef = 50ADFF8B201F53D600D50D53 /* CapacitorPluginMlKitTextRecognitionPlugin.h */; settings = {ATTRIBUTES = (Public, ); }; };
@@ -29,7 +28,6 @@
2928
/* End PBXContainerItemProxy section */
3029

3130
/* Begin PBXFileReference section */
32-
2F98D68124C9AAE400613A4C /* CapacitorPluginMlKitTextRecognition.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CapacitorPluginMlKitTextRecognition.swift; sourceTree = "<group>"; };
3331
3B2A61DA5A1F2DD4F959604D /* Pods_Plugin.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Plugin.framework; sourceTree = BUILT_PRODUCTS_DIR; };
3432
50ADFF88201F53D600D50D53 /* Plugin.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Plugin.framework; sourceTree = BUILT_PRODUCTS_DIR; };
3533
50ADFF8B201F53D600D50D53 /* CapacitorPluginMlKitTextRecognitionPlugin.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = CapacitorPluginMlKitTextRecognitionPlugin.h; sourceTree = "<group>"; };
@@ -93,7 +91,6 @@
9391
isa = PBXGroup;
9492
children = (
9593
50E1A94720377CB70090CE1A /* CapacitorPluginMlKitTextRecognitionPlugin.swift */,
96-
2F98D68124C9AAE400613A4C /* CapacitorPluginMlKitTextRecognition.swift */,
9794
50ADFF8B201F53D600D50D53 /* CapacitorPluginMlKitTextRecognitionPlugin.h */,
9895
50ADFFA72020EE4F00D50D53 /* CapacitorPluginMlKitTextRecognitionPlugin.m */,
9996
50ADFF8C201F53D600D50D53 /* Info.plist */,
@@ -295,7 +292,6 @@
295292
"${BUILT_PRODUCTS_DIR}/GoogleToolboxForMac/GoogleToolboxForMac.framework",
296293
"${BUILT_PRODUCTS_DIR}/GoogleUtilities/GoogleUtilities.framework",
297294
"${BUILT_PRODUCTS_DIR}/PromisesObjC/FBLPromises.framework",
298-
"${BUILT_PRODUCTS_DIR}/Protobuf/Protobuf.framework",
299295
"${BUILT_PRODUCTS_DIR}/nanopb/nanopb.framework",
300296
);
301297
name = "[CP] Embed Pods Frameworks";
@@ -307,7 +303,6 @@
307303
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/GoogleToolboxForMac.framework",
308304
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/GoogleUtilities.framework",
309305
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/FBLPromises.framework",
310-
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/Protobuf.framework",
311306
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/nanopb.framework",
312307
);
313308
runOnlyForDeploymentPostprocessing = 0;
@@ -359,7 +354,6 @@
359354
buildActionMask = 2147483647;
360355
files = (
361356
50E1A94820377CB70090CE1A /* CapacitorPluginMlKitTextRecognitionPlugin.swift in Sources */,
362-
2F98D68224C9AAE500613A4C /* CapacitorPluginMlKitTextRecognition.swift in Sources */,
363357
50ADFFA82020EE4F00D50D53 /* CapacitorPluginMlKitTextRecognitionPlugin.m in Sources */,
364358
);
365359
runOnlyForDeploymentPostprocessing = 0;

‎ios/Plugin/CapacitorPluginMlKitTextRecognitionPlugin.swift

+29-1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,31 @@ import MLKitTextRecognition
88
*/
99
@objc(CapacitorPluginMlKitTextRecognitionPlugin)
1010
public class CapacitorPluginMlKitTextRecognitionPlugin: CAPPlugin {
11+
func parseCornerPointsToJsObject(cornerPoints: [NSValue]) -> Dictionary<String, Any> {
12+
// let res = NSMutableArray();
13+
// res.add(
14+
return [
15+
"topLeft": [
16+
"x": cornerPoints[0].cgPointValue.x,
17+
"y": cornerPoints[0].cgPointValue.y,
18+
],
19+
"topRight": [
20+
"x": cornerPoints[1].cgPointValue.x,
21+
"y": cornerPoints[1].cgPointValue.y,
22+
],
23+
"bottomRight": [
24+
"x": cornerPoints[2].cgPointValue.x,
25+
"y": cornerPoints[2].cgPointValue.y,
26+
],
27+
"bottomLeft": [
28+
"x": cornerPoints[0].cgPointValue.x,
29+
"y": cornerPoints[0].cgPointValue.y,
30+
]
31+
];
32+
// ]);
33+
// return res;
34+
}
35+
1136
@objc func detectText(_ call: CAPPluginCall) {
1237
guard let encodedImage = call.getString("base64Image") else {
1338
call.reject("No image is given!")
@@ -47,7 +72,8 @@ public class CapacitorPluginMlKitTextRecognitionPlugin: CAPPlugin {
4772
"right": element.frame.maxX,
4873
"bottom": element.frame.minY,
4974
],
50-
"recognizedLanguage": element.recognizedLanguages.first?.languageCode ?? ""
75+
"recognizedLanguage": element.recognizedLanguages.first?.languageCode ?? "",
76+
"cornerPoints": self.parseCornerPointsToJsObject(cornerPoints: element.cornerPoints)
5177
])
5278
}
5379

@@ -60,6 +86,7 @@ public class CapacitorPluginMlKitTextRecognitionPlugin: CAPPlugin {
6086
"bottom": line.frame.minY,
6187
],
6288
"recognizedLanguage": line.recognizedLanguages.first?.languageCode ?? "",
89+
"cornerPoints": self.parseCornerPointsToJsObject(cornerPoints: line.cornerPoints),
6390
"elements": elements
6491
])
6592
}
@@ -73,6 +100,7 @@ public class CapacitorPluginMlKitTextRecognitionPlugin: CAPPlugin {
73100
"bottom": textBlock.frame.minY,
74101
],
75102
"recognizedLanguage": textBlock.recognizedLanguages.first?.languageCode ?? "",
103+
"cornerPoints": self.parseCornerPointsToJsObject(cornerPoints: textBlock.cornerPoints),
76104
"lines": lines
77105
])
78106
}

‎ios/Podfile

+12-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ def capacitor_pods
55
use_frameworks!
66
pod 'Capacitor', :path => '../node_modules/@capacitor/ios'
77
pod 'CapacitorCordova', :path => '../node_modules/@capacitor/ios'
8-
pod 'GoogleMLKit/TextRecognition', '5.0.0'
8+
pod 'GoogleMLKit/TextRecognition', '6.0.0'
99
end
1010

1111
target 'Plugin' do
@@ -15,3 +15,14 @@ end
1515
target 'PluginTests' do
1616
capacitor_pods
1717
end
18+
19+
post_install do |installer|
20+
installer.pods_project.targets.each do |target|
21+
target.build_configurations.each do |config|
22+
# Build for all architectures
23+
config.build_settings['ONLY_ACTIVE_ARCH'] = 'NO'
24+
# Exclude arm64 architecture for the iOS simulator
25+
config.build_settings['EXCLUDED_ARCHS[sdk=iphonesimulator*]'] = 'arm64'
26+
end
27+
end
28+
end

0 commit comments

Comments
 (0)