Skip to content

Commit cf106b0

Browse files
dhchenxdhchenx
committed
updated
1 parent 7bea091 commit cf106b0

File tree

85 files changed

+1581
-267
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

85 files changed

+1581
-267
lines changed

‎LICENSE.txt renamed to ‎LICENSE

+10-8
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
1-
Copyright (c) 2016 The Python Packaging Authority (PyPA)
1+
MIT License
22

3-
Permission is hereby granted, free of charge, to any person obtaining a copy of
4-
this software and associated documentation files (the "Software"), to deal in
5-
the Software without restriction, including without limitation the rights to
6-
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
7-
of the Software, and to permit persons to whom the Software is furnished to do
8-
so, subject to the following conditions:
3+
Copyright (c) 2022 The Python Packaging Authority
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
911

1012
The above copyright notice and this permission notice shall be included in all
1113
copies or substantial portions of the Software.
@@ -16,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
1618
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
1719
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
1820
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
19-
SOFTWARE.
21+
SOFTWARE.

‎MANIFEST.in

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,8 @@
1-
include pyproject.toml
2-
31
# Include the README
42
include *.md
53

64
# Include the license file
7-
include LICENSE.txt
5+
include LICENSE
86

97
# Include setup.py
108
include setup.py

‎README.md

+3-3
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
A light-weight Python library to extract, fuse and store multimodal features for deep learning.
44

55
## Objectives
6-
1. To extract and fuse various features from multimodal datasets in a rapid and easy manner;
6+
1. To extract, store and fuse various features from multimodal datasets in a rapid and easy manner;
77
2. To provide a common foundation framework for storage and retrieving of multimodal data.
88

99
## Modalities
@@ -15,7 +15,7 @@ The modalities to support include:
1515
5. Cross-modality between above
1616

1717
## Usage
18-
A toy example showing how to build a multimodal features library is here:
18+
A toy example showing how to build a multimodal feature (MMF) library is here:
1919

2020
```python
2121
from mmkfeatures.fusion.mm_features_lib import MMFeaturesLib
@@ -47,7 +47,7 @@ if __name__ == "__main__":
4747
```
4848

4949
## Credits
50-
The project's source codes come from various open-source projects, we will include a list of their contribution and our improvement.
50+
The project's source codes come from various open-source projects, we will include a list of their contributions and our improvement.
5151

5252
1. [A2Zadeh/CMU-MultimodalSDK](https://github.com/A2Zadeh/CMU-MultimodalSDK)
5353
2. [aishoot/Speech_Feature_Extraction](https://github.com/aishoot/Speech_Feature_Extraction)

‎examples/birds_features_lib/evaluate/step3_use_index.py

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
from mmkfeatures.fusion.mm_features_lib import MMFeaturesLib
2+
import time
3+
4+
mmf_file=f"../datasets/birds.mmf"
5+
6+
def get_exact_match_ratio(lib,result):
7+
num_match = 0
8+
for key in result:
9+
content = lib.get_content_by_id(key)
10+
text = content["text"][()].decode("utf-8", "ignore")
11+
# print(key, text)
12+
if query_str in text:
13+
num_match += 1
14+
print("p = ", round(num_match * 1.0 / len(result), 4))
15+
return round(num_match * 1.0 / len(result), 4)
16+
17+
list_result=[]
18+
start_time=time.time()
19+
20+
21+
print("loading mmf files...")
22+
birds_lib=MMFeaturesLib(file_path=mmf_file)
23+
24+
time_load=time.time()-start_time
25+
list_result.append(("load",time_load))
26+
27+
print("creating plain text index...")
28+
start_time=time.time()
29+
birds_lib.to_index_file("text","../datasets/text.index",index_type="brutal_force")
30+
time_brutal_force=time.time()-start_time
31+
32+
33+
list_result.append(("brutal force indexing",time_brutal_force))
34+
35+
print("creating inverted index....")
36+
start_time=time.time()
37+
birds_lib.to_index_file("text","../datasets/text_inverted.index",index_type="inverted_index")
38+
time_inverted=time.time()-start_time
39+
list_result.append(("inverted indexing",time_inverted))
40+
41+
print("creating positional text...")
42+
start_time=time.time()
43+
birds_lib.to_index_file("text","../datasets/text_positional.index",index_type="positional_index")
44+
time_positional=time.time()-start_time
45+
list_result.append(("positional indexing",time_positional))
46+
47+
# start to perform search test
48+
query_str="large brown wings"
49+
50+
print("searching plain index test....")
51+
start_time=time.time()
52+
result_bf=birds_lib.search_index(index_file_path="../datasets/text.index",query=query_str,search_type="brutal_force")
53+
print(result_bf)
54+
search_time_brutal=time.time()-start_time
55+
list_result.append(("brutal force searching",search_time_brutal,get_exact_match_ratio(birds_lib,result_bf)))
56+
57+
print("searching inverted index test....")
58+
start_time=time.time()
59+
result_bf=birds_lib.search_index(index_file_path="../datasets/text_inverted.index",query=query_str,search_type="inverted_index")
60+
print(result_bf)
61+
search_time_inverted=time.time()-start_time
62+
list_result.append(("inverted index searching",search_time_inverted,get_exact_match_ratio(birds_lib,result_bf)))
63+
64+
print("searching positional index test....")
65+
start_time=time.time()
66+
result_bf=birds_lib.search_index(index_file_path="../datasets/text_positional.index",query=query_str,search_type="positional_index")
67+
print(result_bf)
68+
search_time_positional=time.time()-start_time
69+
list_result.append(("positional index searching",search_time_positional,get_exact_match_ratio(birds_lib,result_bf)))
70+
print()
71+
print("Indexing method\tTime cost\tExact match ratio")
72+
for result in list_result:
73+
if len(result)==3:
74+
print(f"{result[0]}\t{result[1]}\t{result[2]}")
75+
76+
print()
77+
print("Loading method\tTime cost")
78+
for result in list_result:
79+
if len(result)==2:
80+
print(f"{result[0]}\t{result[1]}")
81+
82+

‎examples/birds_features_lib/evaluate/step4_create_image_index.py

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
from mmkfeatures.fusion.mm_features_lib import MMFeaturesLib
2+
from mmkfeatures.image.color_descriptor import ColorDescriptor
3+
import numpy as np
4+
import cv2
5+
import pickle
6+
from tqdm import tqdm
7+
import time
8+
9+
mmf_file=f"../datasets/birds_raw.mmf"
10+
11+
feature_lib=MMFeaturesLib(file_path=mmf_file)
12+
13+
data=feature_lib.get_data()
14+
15+
cd = ColorDescriptor((8, 12, 3))
16+
17+
list_features=[]
18+
19+
print(data.keys())
20+
start_time=time.time()
21+
f_out=open("../datasets/image.index","w")
22+
for cid in tqdm(data.keys()):
23+
item=data[cid]
24+
imgs=item["objects"]
25+
for img_id in imgs:
26+
# print(image)
27+
img=imgs[img_id][()]
28+
# print(img)
29+
# print(type(img))
30+
features = cd.describe(img)
31+
features = [str(f) for f in features]
32+
# print(feature)
33+
feature_str=cid+","+",".join(features)
34+
f_out.write(feature_str+"\n")
35+
f_out.close()
36+
end_time=time.time()
37+
time_cost=end_time-start_time
38+
print("creating image index: ",time_cost)

‎examples/birds_features_lib/evaluate/step5_search_image_index.py

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import time
2+
3+
from mmkfeatures.fusion.mm_features_lib import MMFeaturesLib
4+
from mmkfeatures.image.color_descriptor import ColorDescriptor
5+
import numpy as np
6+
import cv2
7+
import pickle
8+
from tqdm import tqdm
9+
from mmkfeatures.image.image_searcher import Searcher
10+
11+
mmf_file=f"../datasets/birds_raw.mmf"
12+
13+
feature_lib=MMFeaturesLib(file_path=mmf_file)
14+
15+
data=feature_lib.get_data()
16+
17+
18+
# initialize the image descriptor
19+
cd = ColorDescriptor((8, 12, 3))
20+
21+
img_path="../datasets/CUB_200_2011/images/005.Crested_Auklet/Crested_Auklet_0001_794941.jpg"
22+
index_path="../datasets/image.index"
23+
24+
print("Searching....")
25+
26+
27+
# load the query image and describe it
28+
query = cv2.imread(img_path)
29+
features = cd.describe(query)
30+
# perform the search
31+
start_time=time.time()
32+
searcher = Searcher(index_path)
33+
results = searcher.search(features)
34+
end_time=time.time()
35+
36+
print("query time cost: ",end_time-start_time)
37+
# display the query
38+
cv2.imshow("Query", query)
39+
cv2.waitKey(0)
40+
# loop over the results
41+
for (score, resultID) in results:
42+
print(resultID,score)
43+
image=data[str(resultID)]["objects"]["0"][()]
44+
title=data[str(resultID)]["labels"][()][0]
45+
cv2.imshow(str(title), image)
46+
cv2.waitKey(0)
47+
# load the result image and display it
48+
# result = cv2.imread("datasets/CUB_200_2011/" + resultID)
49+
# cv2.imshow("Result", result)
50+
# cv2.waitKey(0)

‎examples/birds_features_lib/evaluate/step6_search_image_index_simplified.py

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
from mmkfeatures.fusion.mm_features_lib import MMFeaturesLib
2+
from mmkfeatures.image.color_descriptor import ColorDescriptor
3+
import cv2
4+
import time
5+
# load an existing multimodal feature lib
6+
mmf_file=f"../datasets/birds_raw.mmf"
7+
feature_lib=MMFeaturesLib(file_path=mmf_file)
8+
data=feature_lib.get_data()
9+
10+
# set test image and index file's path
11+
test_img_path="../datasets/CUB_200_2011/images/005.Crested_Auklet/Crested_Auklet_0001_794941.jpg"
12+
index_path="../datasets/image.index"
13+
14+
# create index
15+
feature_lib.to_obj_index(index_file=index_path,obj_field="objects",index_type="color_descriptor")
16+
17+
# query index by color_descriptor
18+
cd = ColorDescriptor((8, 12, 3))
19+
query_image = cv2.imread(test_img_path)
20+
query_features = cd.describe(query_image)
21+
start_time=time.time()
22+
search_results=feature_lib.search_obj_index(index_file=index_path,features=query_features)
23+
end_time=time.time()
24+
print("simplified time cost: ",end_time-start_time)
25+
26+
# loop over the results
27+
for (score, resultID) in search_results:
28+
print(resultID,score)
29+
content=feature_lib.get_content_by_id(resultID)
30+
# print(content)
31+
image=content["objects"]["0"][()]
32+
title=content["labels"][()][0]
33+
cv2.imshow(str(title), image)
34+
cv2.waitKey(0)
35+

‎examples/birds_features_lib/evaluate/test_inverted_index.py

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
from mmkfeatures.fusion.mm_features_lib import MMFeaturesLib
2+
import time
3+
4+
mmf_file=f"../datasets/birds.mmf"
5+
list_result=[]
6+
start_time=time.time()
7+
8+
print("loading mmf files...")
9+
birds_lib=MMFeaturesLib(file_path=mmf_file)
10+
11+
# creating inverted index
12+
birds_lib.to_index_file("text","../datasets/text_inverted.index",index_type="inverted_index")
13+
time_inverted=time.time()-start_time
14+
list_result.append(("inverted indexing",time_inverted))
15+
16+
print("time cost of creating inverted: ",time.time()-start_time)
17+
18+
# start to perform search test
19+
query_str="large brown wings"
20+
21+
print("searching inverted index test....")
22+
start_time=time.time()
23+
result_bf=birds_lib.search_index(index_file_path="../datasets/text_inverted.index",query=query_str,search_type="inverted_index")
24+
print(result_bf)
25+
26+
print("time cost of search inverted: ",time.time()-start_time)
27+
28+
num_match=0
29+
for key in result_bf:
30+
content=birds_lib.get_content_by_id(key)
31+
text=content["text"][()].decode("utf-8","ignore")
32+
print(key,text)
33+
if query_str in text:
34+
num_match+=1
35+
print("p = ",round(num_match*1.0/len(result_bf),4))

‎examples/birds_features_lib/evaluate/test_positional_index.py

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
from mmkfeatures.fusion.mm_features_lib import MMFeaturesLib
2+
import time
3+
4+
mmf_file=f"../datasets/birds.mmf"
5+
list_result=[]
6+
start_time=time.time()
7+
8+
print("loading mmf files...")
9+
birds_lib=MMFeaturesLib(file_path=mmf_file)
10+
print("loading ",time.time()-start_time)
11+
# creating inverted index
12+
print("creating positional text...")
13+
start_time=time.time()
14+
birds_lib.to_index_file("text","../datasets/text_positional.index",index_type="positional_index")
15+
16+
time_positional=time.time()-start_time
17+
list_result.append(("positional indexing",time_positional))
18+
19+
# start to perform search test
20+
query_str="large brown wings"
21+
22+
23+
print("searching positional index test....")
24+
start_time=time.time()
25+
result_bf=birds_lib.search_index(index_file_path="../datasets/text_positional.index",query=query_str,search_type="positional_index")
26+
print(result_bf)
27+
28+
search_time_positional=time.time()-start_time
29+
30+
print("search time: ",search_time_positional)
31+
32+
# show results
33+
num_match=0
34+
for key in result_bf:
35+
content=birds_lib.get_content_by_id(key)
36+
text=content["text"][()].decode("utf-8","ignore")
37+
print(key,text)
38+
if query_str in text:
39+
num_match+=1
40+
41+
print("p = ",round(num_match*1.0/len(result_bf),4))
42+
43+
# export key values
44+
45+
# birds_lib.export_key_values("text",save_path="text.csv")
46+

‎examples/birds_features_lib/step1_create_bird_lib.py

-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
import os
2-
import sys
3-
import csv
41
from tqdm import tqdm
52
import cv2
63
import numpy as np

‎examples/birds_features_lib/step5_search_image_index.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,15 @@
2727
searcher = Searcher(index_path)
2828
results = searcher.search(features)
2929
# display the query
30-
# cv2.imshow("Query", query)
31-
# cv2.waitKey(0)
30+
cv2.imshow("Query", query)
31+
cv2.waitKey(0)
3232

3333
# loop over the results
3434
for (score, resultID) in results:
3535
print(resultID,score)
3636
image=data[str(resultID)]["objects"]["0"][()]
3737
title=data[str(resultID)]["labels"][()][0]
38-
cv2.imshow(title, image)
38+
cv2.imshow(str(title), image)
3939
cv2.waitKey(0)
4040
# load the result image and display it
4141
# result = cv2.imread("datasets/CUB_200_2011/" + resultID)

0 commit comments

Comments
 (0)