Skip to content

Commit 40d1527

Browse files
matyhtfmatyhtf
committed
add bloom_filter
1 parent 9257da9 commit 40d1527

File tree

11 files changed

+1401
-0
lines changed

11 files changed

+1401
-0
lines changed

Diff for: ‎examples/bloom_filter/.gitignore

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Created by .ignore support plugin (hsz.mobi)
2+
### C++ template
3+
# Compiled Object files
4+
*.slo
5+
*.lo
6+
*.o
7+
*.obj
8+
9+
# Precompiled Headers
10+
*.gch
11+
*.pch
12+
13+
# Compiled Dynamic libraries
14+
*.so
15+
*.dylib
16+
*.dll
17+
18+
# Fortran module files
19+
*.mod
20+
*.smod
21+
22+
# Compiled Static libraries
23+
*.lai
24+
*.la
25+
*.a
26+
*.lib
27+
28+
# Executables
29+
*.exe
30+
*.out
31+
*.app
32+
33+
/.cproject
34+
/.project
35+
/.settings/
36+
/.idea
37+
/tests/bf.data

Diff for: ‎examples/bloom_filter/BloomFilter.cpp

+246
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
#include "phpx.h"
2+
3+
extern "C"
4+
{
5+
#include <php_swoole.h>
6+
#include <swoole.h>
7+
extern void MurmurHash3_x64_128(const void * key, const int len, const uint32_t seed, void *out);
8+
extern void SpookyHash128(const void *key, size_t len, uint64_t seed1, uint64_t seed2, uint64_t *hash1, uint64_t *hash2);
9+
}
10+
11+
#include <iostream>
12+
13+
using namespace php;
14+
using namespace std;
15+
16+
struct BloomFilterObject
17+
{
18+
size_t capacity;
19+
char *array;
20+
uint32_t k_num;
21+
uint64_t bit_num;
22+
uint64_t *hashes;
23+
swLock lock;
24+
};
25+
26+
#define RESOURCE_NAME "BloomFilterResource"
27+
#define PROPERTY_NAME "ptr"
28+
29+
static void compute_hashes(uint32_t k_num, char *key, size_t len, uint64_t *hashes)
30+
{
31+
uint64_t out[2];
32+
MurmurHash3_x64_128(key, len, 0, out);
33+
34+
hashes[0] = out[0];
35+
hashes[1] = out[1];
36+
37+
uint64_t *hash1 = out;
38+
uint64_t *hash2 = hash1 + 1;
39+
SpookyHash128(key, len, 0, 0, hash1, hash2);
40+
41+
hashes[2] = out[0];
42+
hashes[3] = out[1];
43+
44+
for (uint32_t i = 4; i < k_num; i++)
45+
{
46+
hashes[i] = hashes[1] + ((i * hashes[3]) % 18446744073709551557U);
47+
}
48+
}
49+
50+
static void BloomFilterResDtor(zend_resource *res)
51+
{
52+
BloomFilterObject *bf = static_cast<BloomFilterObject *>(res->ptr);
53+
efree(bf->hashes);
54+
bf->lock.free(&bf->lock);
55+
sw_shm_free(bf);
56+
}
57+
58+
static PHPX_METHOD(BloomFilter, __construct)
59+
{
60+
long capacity = args[0].toInt();
61+
if (capacity <= 0)
62+
{
63+
capacity = 65536;
64+
}
65+
66+
uint32_t k_num = 2;
67+
if (args.exists(1))
68+
{
69+
k_num = (uint32_t) args[1].toInt();
70+
}
71+
72+
BloomFilterObject *bf = (BloomFilterObject *) sw_shm_malloc(sizeof(BloomFilterObject) + capacity);
73+
if (bf == NULL)
74+
{
75+
throwException("RuntimeException", "sw_shm_malloc() failed.");
76+
}
77+
bf->capacity = capacity;
78+
bf->array = (char *) (bf + 1);
79+
bzero(bf->array, bf->capacity);
80+
81+
bf->hashes = (uint64_t*) ecalloc(k_num, sizeof(uint64_t));
82+
if (bf->hashes == NULL)
83+
{
84+
throwException("RuntimeException", "ecalloc() failed.");
85+
}
86+
bf->bit_num = bf->capacity * 8;
87+
bf->k_num = k_num;
88+
89+
if (swRWLock_create(&bf->lock, 1) < 0)
90+
{
91+
throwException("RuntimeException", "swRWLock_create() failed.");
92+
}
93+
94+
_this.oSet(PROPERTY_NAME, RESOURCE_NAME, bf);
95+
}
96+
97+
PHPX_METHOD(BloomFilter, has)
98+
{
99+
BloomFilterObject *bf = _this.oGet<BloomFilterObject>(PROPERTY_NAME, RESOURCE_NAME);
100+
auto key = args[0];
101+
compute_hashes(bf->k_num, key.toCString(), key.length(), bf->hashes);
102+
103+
uint32_t i;
104+
uint32_t n;
105+
bool miss;
106+
107+
bf->lock.lock_rd(&bf->lock);
108+
for (i = 0; i < bf->k_num; i++)
109+
{
110+
n = bf->hashes[i] % bf->bit_num;
111+
miss = !(bf->array[n / 8] & (1 << (n % 8)));
112+
if (miss)
113+
{
114+
bf->lock.unlock(&bf->lock);
115+
retval = false;
116+
return;
117+
}
118+
}
119+
bf->lock.unlock(&bf->lock);
120+
retval = true;
121+
}
122+
123+
static PHPX_METHOD(BloomFilter, add)
124+
{
125+
BloomFilterObject *bf = _this.oGet<BloomFilterObject>(PROPERTY_NAME, RESOURCE_NAME);
126+
auto key = args[0];
127+
compute_hashes(bf->k_num, key.toCString(), key.length(), bf->hashes);
128+
129+
uint32_t i;
130+
uint32_t n;
131+
132+
bf->lock.lock(&bf->lock);
133+
for (i = 0; i < bf->k_num; i++)
134+
{
135+
n = bf->hashes[i] % bf->bit_num;
136+
bf->array[n / 8] |= (1 << (n % 8));
137+
}
138+
bf->lock.unlock(&bf->lock);
139+
}
140+
141+
static PHPX_METHOD(BloomFilter, clear)
142+
{
143+
BloomFilterObject *bf = _this.oGet<BloomFilterObject>(PROPERTY_NAME, RESOURCE_NAME);
144+
bf->lock.lock(&bf->lock);
145+
bzero(bf->array, bf->capacity);
146+
bf->lock.unlock(&bf->lock);
147+
}
148+
149+
static PHPX_METHOD(BloomFilter, dump)
150+
{
151+
BloomFilterObject *bf = _this.oGet<BloomFilterObject>(PROPERTY_NAME, RESOURCE_NAME);
152+
auto file = args[0].toCString();
153+
154+
bf->lock.lock(&bf->lock);
155+
int fd = open(file, O_CREAT | O_WRONLY, 0644);
156+
if (fd < 0)
157+
{
158+
fail: bf->lock.unlock(&bf->lock);
159+
retval = false;
160+
if (fd >= 0)
161+
{
162+
close(fd);
163+
unlink(file);
164+
}
165+
return;
166+
}
167+
if (swoole_sync_writefile(fd, &bf->capacity, sizeof(bf->capacity)) < 0)
168+
{
169+
goto fail;
170+
}
171+
if (swoole_sync_writefile(fd, bf->array, bf->capacity) < 0)
172+
{
173+
goto fail;
174+
}
175+
bf->lock.unlock(&bf->lock);
176+
close(fd);
177+
retval = true;
178+
}
179+
180+
static PHPX_METHOD(BloomFilter, load)
181+
{
182+
auto file = args[0].toCString();
183+
int fd = open(file, O_RDONLY);
184+
if (fd < 0)
185+
{
186+
fail: retval = false;
187+
if (fd >= 0)
188+
{
189+
close(fd);
190+
}
191+
return;
192+
}
193+
194+
size_t capacity;
195+
if (swoole_sync_readfile(fd, &capacity, sizeof(capacity)) < 0)
196+
{
197+
goto fail;
198+
}
199+
200+
long filesize = swoole_file_size(file);
201+
if (filesize < 0 || filesize < capacity + sizeof(capacity))
202+
{
203+
goto fail;
204+
}
205+
206+
auto o = newObject("BloomFilter", (long) capacity);
207+
BloomFilterObject *bf = o.oGet<BloomFilterObject>(PROPERTY_NAME, RESOURCE_NAME);
208+
if (swoole_sync_readfile(fd, bf->array, capacity) < 0)
209+
{
210+
goto fail;
211+
}
212+
close(fd);
213+
retval = o;
214+
}
215+
216+
PHPX_EXTENSION()
217+
{
218+
Extension *extension = new Extension("BloomFilter", "1.0.1");
219+
220+
extension->onStart = [extension]() noexcept
221+
{
222+
extension->registerConstant("BLOOMFILTER_VERSION", 10001);
223+
224+
Class *c = new Class("BloomFilter");
225+
c->addMethod(PHPX_ME(BloomFilter, __construct), CONSTRUCT);
226+
c->addMethod(PHPX_ME(BloomFilter, add));
227+
c->addMethod(PHPX_ME(BloomFilter, has));
228+
c->addMethod(PHPX_ME(BloomFilter, clear));
229+
c->addMethod(PHPX_ME(BloomFilter, load), STATIC);
230+
c->addMethod(PHPX_ME(BloomFilter, dump));
231+
232+
extension->registerClass(c);
233+
extension->registerResource(RESOURCE_NAME, BloomFilterResDtor);
234+
};
235+
236+
extension->require("swoole");
237+
238+
extension->info({"BloomFilter support", "enabled"},
239+
{
240+
{"author", "Tianfeng Han"},
241+
{"version", extension->version},
242+
{"date", "2018-01-10"},
243+
});
244+
245+
return extension;
246+
}

Diff for: ‎examples/bloom_filter/Makefile

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
PHP_INCLUDE = `php-config --includes`
2+
PHP_LIBS = `php-config --libs`
3+
PHP_LDFLAGS = `php-config --ldflags`
4+
PHP_INCLUDE_DIR = `php-config --include-dir`
5+
PHP_EXTENSION_DIR = `php-config --extension-dir`
6+
7+
BloomFilter.so: BloomFilter.cpp spooky.o MurmurHash3.o
8+
c++ -DHAVE_CONFIG_H -g -o BloomFilter.so -O2 -fPIC -shared BloomFilter.cpp spooky.o MurmurHash3.o -std=c++11 -lphpx ${PHP_INCLUDE} -I${PHP_INCLUDE_DIR} -I${PHP_INCLUDE_DIR}/ext/swoole -I${PHP_INCLUDE_DIR}/ext/swoole/include ${PHP_LIBS} ${PHP_LDFLAGS}
9+
spooky.o: deps/spookyhash/spooky.cpp
10+
c++ -c -fPIC -O2 -g -o spooky.o deps/spookyhash/spooky.cpp
11+
MurmurHash3.o: deps/murmurhash/MurmurHash3.cpp
12+
c++ -c -fPIC -O2 -g -o MurmurHash3.o deps/murmurhash/MurmurHash3.cpp
13+
install: BloomFilter.so
14+
cp BloomFilter.so ${PHP_EXTENSION_DIR}/
15+
clean:
16+
rm *.o
17+
rm *.so
18+

Diff for: ‎examples/bloom_filter/README.md

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# BloomFilter
2+
基于共享内存的 `BloomFilter` 库。
3+
4+
* 部分代码借鉴自 <https://github.com/armon/bloomd> 项目
5+
* 参考资料
6+
* <https://my.oschina.net/kiwivip/blog/133498>
7+
* <http://blog.csdn.net/jiaomeng/article/details/1495500>
8+
* <http://blog.csdn.net/hguisu/article/details/7866173>
9+
10+
## Require
11+
12+
* `SWOOLE`
13+
* `PHP-X`
14+
* `PHP >= 7.0`
15+
16+
## 安装使用
17+
18+
```bash
19+
make && make install
20+
```
21+
22+
到`php.ini`添加`extension=BloomFilter.so`
23+
24+
## 示例
25+
26+
```php
27+
<?php
28+
$b = new BloomFilter(1024);
29+
echo "key hello\n";
30+
$b->add("hello");
31+
$b->add("world");
32+
$b->add("rango");
33+
var_dump($b->has("test"));
34+
var_dump($b->has("hello"));
35+
var_dump($b->has("world"));
36+
var_dump($b->has(" world "));
37+
var_dump($b->has("rango"));
38+
var_dump($b->has("xxxxxxxxxxxxxxxxxxxxxxx"));
39+
var_dump($b->has("rango12"));
40+
var_dump($b->has("me"));
41+
$b->dump(__DIR__."/bf.data");
42+
```

0 commit comments

Comments
 (0)