Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support Python 3 #28

Open
wants to merge 16 commits into
base: develop
Choose a base branch
from
Open

Support Python 3 #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
c411ae6
Run CI on Python 3.5 and 3.6
jakirkham Jun 12, 2017
54bb82e
Skip using system `site-packages`
jakirkham Jun 12, 2017
355f1bb
Don't use `apt-get` to install `numpy` and `scipy`
jakirkham Jun 12, 2017
d8bf814
Install NumPy and SciPy from pip
jakirkham Jun 12, 2017
82014d1
Disable `sudo` as it is no longer needed
jakirkham Jun 12, 2017
73cbdbf
Fix Python 2/3 compatibility issues with pickle
jakirkham Jun 12, 2017
d85e8da
Use print function for Python 2/3 compatibility
jakirkham Jun 12, 2017
925a148
Require binaries for NumPy and SciPy
jakirkham Jun 12, 2017
ccc987a
`METH_KEYWORDS` -> `METH_VARARGS | METH_KEYWORDS`
jakirkham Jun 12, 2017
d20f038
Set input_indices arguments to METH_VARARGS
jakirkham Jun 12, 2017
9348906
Update pip on Travis CI
jakirkham Jun 12, 2017
44f4c79
Use `^` with masks instead of `-`
jakirkham Jun 12, 2017
bdfb406
Handle unicode regularizer
jakirkham Jun 12, 2017
6842b96
Improve error checking in regularizer
jakirkham Jun 12, 2017
03e5885
Convert `range` to `list` on Python 3
jakirkham Jun 12, 2017
296a76e
Get the `pvalue` from SciPy's `ks_2samp`
jakirkham Jun 12, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
sudo: false
language: python
python:
- "2.7"
virtualenv:
system_site_packages: true
before_install:
- sudo apt-get -qq install python-numpy python-scipy
- "3.5"
- "3.6"
install:
- python -m pip install -U pip
- pip install --only-binary=":all:" numpy scipy
- cd code/liblbfgs
- ./autogen.sh
- ./configure --enable-sse2
Expand Down
18 changes: 9 additions & 9 deletions code/cmt/python/src/module.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -762,10 +762,10 @@ static PyGetSetDef PatchModel_getset[] = {
};

static PyMethodDef PatchModel_methods[] = {
{"loglikelihood", (PyCFunction)PatchModel_loglikelihood, METH_KEYWORDS, 0},
{"loglikelihood", (PyCFunction)PatchModel_loglikelihood, METH_VARARGS | METH_KEYWORDS, 0},
{"input_mask", (PyCFunction)PatchModel_input_mask, METH_VARARGS, 0},
{"output_mask", (PyCFunction)PatchModel_output_mask, METH_VARARGS, 0},
{"input_indices", (PyCFunction)PatchModel_input_indices, 0, 0},
{"input_indices", (PyCFunction)PatchModel_input_indices, METH_VARARGS, 0},
{0}
};

Expand Down Expand Up @@ -825,9 +825,9 @@ static PyGetSetDef PatchMCBM_getset[] = {
};

static PyMethodDef PatchMCBM_methods[] = {
{"initialize", (PyCFunction)PatchMCBM_initialize, METH_KEYWORDS, PatchMCBM_initialize_doc},
{"train", (PyCFunction)PatchMCBM_train, METH_KEYWORDS, PatchMCBM_train_doc},
{"preconditioner", (PyCFunction)PatchMCBM_preconditioner, METH_VARARGS, 0},
{"initialize", (PyCFunction)PatchMCBM_initialize, METH_VARARGS | METH_KEYWORDS, PatchMCBM_initialize_doc},
{"train", (PyCFunction)PatchMCBM_train, METH_VARARGS | METH_KEYWORDS, PatchMCBM_train_doc},
{"preconditioner", (PyCFunction)PatchMCBM_preconditioner, METH_VARARGS | METH_VARARGS, 0},
{"__reduce__", (PyCFunction)PatchMCBM_reduce, METH_NOARGS, PatchMCBM_reduce_doc},
{"__setstate__", (PyCFunction)PatchMCBM_setstate, METH_VARARGS, PatchMCBM_setstate_doc},
{0}
Expand Down Expand Up @@ -889,8 +889,8 @@ static PyGetSetDef PatchMCGSM_getset[] = {
};

static PyMethodDef PatchMCGSM_methods[] = {
{"initialize", (PyCFunction)PatchMCGSM_initialize, METH_KEYWORDS, PatchMCGSM_initialize_doc},
{"train", (PyCFunction)PatchMCGSM_train, METH_KEYWORDS, PatchMCGSM_train_doc},
{"initialize", (PyCFunction)PatchMCGSM_initialize, METH_VARARGS | METH_KEYWORDS, PatchMCGSM_initialize_doc},
{"train", (PyCFunction)PatchMCGSM_train, METH_VARARGS | METH_KEYWORDS, PatchMCGSM_train_doc},
{"preconditioner", (PyCFunction)PatchMCGSM_preconditioner, METH_VARARGS, 0},
{"__reduce__", (PyCFunction)PatchMCGSM_reduce, METH_NOARGS, PatchMCGSM_reduce_doc},
{"__setstate__", (PyCFunction)PatchMCGSM_setstate, METH_VARARGS, PatchMCGSM_setstate_doc},
Expand Down Expand Up @@ -1124,8 +1124,8 @@ static PyGetSetDef FVBN_getset[] = {
};

static PyMethodDef FVBN_methods[] = {
{"initialize", (PyCFunction)FVBN_initialize, METH_KEYWORDS, FVBN_initialize_doc},
{"train", (PyCFunction)FVBN_train, METH_KEYWORDS, FVBN_train_doc},
{"initialize", (PyCFunction)FVBN_initialize, METH_VARARGS | METH_KEYWORDS, FVBN_initialize_doc},
{"train", (PyCFunction)FVBN_train, METH_VARARGS | METH_KEYWORDS, FVBN_train_doc},
{"preconditioner", (PyCFunction)FVBN_preconditioner, METH_VARARGS, 0},
{"__reduce__", (PyCFunction)FVBN_reduce, METH_NOARGS, FVBN_reduce_doc},
{"__setstate__", (PyCFunction)FVBN_setstate, METH_VARARGS, FVBN_setstate_doc},
Expand Down
16 changes: 14 additions & 2 deletions code/cmt/python/src/pyutils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ using std::make_pair;
#define PyInt_FromLong PyLong_FromLong
#define PyInt_AsLong PyLong_AsLong
#define PyInt_Check PyLong_Check
#define PyString_Check PyBytes_Check
#define PyString_Size PyBytes_Size
#define PyString_AsString PyBytes_AsString
#endif
Expand Down Expand Up @@ -378,10 +379,21 @@ Regularizer PyObject_ToRegularizer(PyObject* regularizer) {
Regularizer::Norm norm = Regularizer::L2;

if(r_norm) {
if(PyString_Size(r_norm) != 2)
if(PyUnicode_Check(r_norm))
r_norm = PyUnicode_AsASCIIString(r_norm);

if((r_norm == NULL) || (!PyString_Check(r_norm)) || (PyString_Size(r_norm) != 2)) {
PyErr_Clear();
throw Exception("Regularizer norm should be 'L1' or 'L2'.");
}

char* r_norm_str = PyString_AsString(r_norm);
if(r_norm_str == NULL) {
PyErr_Clear();
throw Exception("Regularizer norm should be 'L1' or 'L2'.");
}

switch(PyString_AsString(r_norm)[1]) {
switch(r_norm_str[1]) {
default:
throw Exception("Regularizer norm should be 'L1' or 'L2'.");

Expand Down
8 changes: 4 additions & 4 deletions code/cmt/python/tests/fvbn_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ def test_fvbn(self):

model = FVBN(8, 8, xmask, ymask)

self.assertLess(max(abs(model.input_mask() - xmask)), 1e-8)
self.assertLess(max(abs(model.output_mask() - ymask)), 1e-8)
self.assertLess(max(abs(model.input_mask() ^ xmask)), 1e-8)
self.assertLess(max(abs(model.output_mask() ^ ymask)), 1e-8)

for i in range(8):
for j in range(8):
Expand Down Expand Up @@ -67,11 +67,11 @@ def test_pickle(self):
tmp_file = mkstemp()[1]

# store model
with open(tmp_file, 'w') as handle:
with open(tmp_file, 'wb') as handle:
dump({'model': model0}, handle)

# load model
with open(tmp_file) as handle:
with open(tmp_file, 'rb') as handle:
model1 = load(handle)['model']

# make sure parameters haven't changed
Expand Down
4 changes: 2 additions & 2 deletions code/cmt/python/tests/glm_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,11 +168,11 @@ def test_glm_pickle(self):
model0.bias = randn()

# store model
with open(tmp_file, 'w') as handle:
with open(tmp_file, 'wb') as handle:
dump({'model': model0}, handle)

# load model
with open(tmp_file) as handle:
with open(tmp_file, 'rb') as handle:
model1 = load(handle)['model']

# make sure parameters haven't changed
Expand Down
4 changes: 2 additions & 2 deletions code/cmt/python/tests/gsm_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,11 @@ def test_pickle(self):
tmp_file = mkstemp()[1]

# store model
with open(tmp_file, 'w') as handle:
with open(tmp_file, 'wb') as handle:
dump({'model': model0}, handle)

# load model
with open(tmp_file) as handle:
with open(tmp_file, 'rb') as handle:
model1 = load(handle)['model']

# make sure parameters haven't changed
Expand Down
38 changes: 19 additions & 19 deletions code/cmt/python/tests/mcbm_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,11 +149,11 @@ def test_pickle(self):
tmp_file = mkstemp()[1]

# store model
with open(tmp_file, 'w') as handle:
with open(tmp_file, 'wb') as handle:
dump({'mcbm': mcbm0}, handle)

# load model
with open(tmp_file) as handle:
with open(tmp_file, 'rb') as handle:
mcbm1 = load(handle)['mcbm']

# make sure parameters haven't changed
Expand All @@ -178,8 +178,8 @@ def test_patchmcbm(self):

model = PatchMCBM(8, 8, xmask, ymask, model=MCBM(sum(xmask), 1))

self.assertLess(max(abs(model.input_mask() - xmask)), 1e-8)
self.assertLess(max(abs(model.output_mask() - ymask)), 1e-8)
self.assertLess(max(abs(model.input_mask() ^ xmask)), 1e-8)
self.assertLess(max(abs(model.output_mask() ^ ymask)), 1e-8)

for i in range(8):
for j in range(8):
Expand All @@ -192,8 +192,8 @@ def test_patchmcbm(self):

model = PatchMCBM(rows, cols, xmask, ymask, order, MCBM(sum(xmask), 1))

self.assertLess(max(abs(model.input_mask() - xmask)), 1e-8)
self.assertLess(max(abs(model.output_mask() - ymask)), 1e-8)
self.assertLess(max(abs(model.input_mask() ^ xmask)), 1e-8)
self.assertLess(max(abs(model.output_mask() ^ ymask)), 1e-8)

for i in range(rows):
for j in range(cols):
Expand All @@ -203,8 +203,8 @@ def test_patchmcbm(self):
model0 = PatchMCBM(rows, cols, max_pcs=3)
model1 = PatchMCBM(rows, cols, model0.input_mask(), model0.output_mask(), model0.order)

self.assertLess(max(abs(model0.input_mask() - model1.input_mask())), 1e-8)
self.assertLess(max(abs(model0.output_mask() - model1.output_mask())), 1e-8)
self.assertLess(max(abs(model0.input_mask() ^ model1.input_mask())), 1e-8)
self.assertLess(max(abs(model0.output_mask() ^ model1.output_mask())), 1e-8)
self.assertLess(max(abs(asarray(model0.order) - asarray(model1.order))), 1e-8)

# test computation of input masks
Expand All @@ -213,7 +213,7 @@ def test_patchmcbm(self):
i, j = model0.order[0]
input_mask = model.input_mask(i, j)
for i, j in model.order[1:]:
self.assertEqual(sum(model.input_mask(i, j) - input_mask), 1)
self.assertEqual(sum(model.input_mask(i, j) ^ input_mask), 1)
input_mask = model.input_mask(i, j)


Expand Down Expand Up @@ -329,11 +329,11 @@ def test_patchmcbm_pickle(self):
tmp_file = mkstemp()[1]

# store model
with open(tmp_file, 'w') as handle:
with open(tmp_file, 'wb') as handle:
dump({'model': model0}, handle)

# load model
with open(tmp_file) as handle:
with open(tmp_file, 'rb') as handle:
model1 = load(handle)['model']

# make sure parameters haven't changed
Expand All @@ -360,19 +360,19 @@ def test_patchmcbm_pickle(self):
tmp_file = mkstemp()[1]

# store model
with open(tmp_file, 'w') as handle:
with open(tmp_file, 'wb') as handle:
dump({'model': model0}, handle)

# load model
with open(tmp_file) as handle:
with open(tmp_file, 'rb') as handle:
model1 = load(handle)['model']

# make sure parameters haven't changed
self.assertEqual(model0.rows, model1.rows)
self.assertEqual(model0.cols, model1.cols)
self.assertLess(max(abs(asarray(model1.order) - asarray(model0.order))), 1e-20)
self.assertLess(max(abs(model0.input_mask() - model1.input_mask())), 1e-20)
self.assertLess(max(abs(model0.output_mask() - model1.output_mask())), 1e-20)
self.assertLess(max(abs(model0.input_mask() ^ model1.input_mask())), 1e-20)
self.assertLess(max(abs(model0.output_mask() ^ model1.output_mask())), 1e-20)

for i in range(model0.rows):
for j in range(model0.cols):
Expand All @@ -389,11 +389,11 @@ def test_patchmcbm_pickle(self):
logLik = mean(model0.loglikelihood(samples))

# store model
with open(tmp_file, 'w') as handle:
with open(tmp_file, 'wb') as handle:
dump({'model': model0}, handle)

# load model
with open(tmp_file) as handle:
with open(tmp_file, 'rb') as handle:
model1 = load(handle)['model']

self.assertAlmostEqual(mean(model1.loglikelihood(samples)), logLik)
Expand All @@ -404,11 +404,11 @@ def test_patchmcbm_pickle(self):
model0 = PatchMCBM(rows, cols, xmask, ymask, order)

# store model
with open(tmp_file, 'w') as handle:
with open(tmp_file, 'wb') as handle:
dump({'model': model0}, handle)

# load model
with open(tmp_file) as handle:
with open(tmp_file, 'rb') as handle:
model1 = load(handle)['model']

for i in range(rows):
Expand Down
26 changes: 13 additions & 13 deletions code/cmt/python/tests/mcgsm_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ def callback(i, mcgsm):
})

# test callback
self.assertTrue(range(cb_iter, max_iter + 1, cb_iter) == count)
self.assertTrue(list(range(cb_iter, max_iter + 1, cb_iter)) == count)



Expand Down Expand Up @@ -174,9 +174,9 @@ def test_mogsm(self):

# generated samples should have the same distribution
for i in range(mogsm.dim):
self.assertTrue(ks_2samp(mogsm_samples[i], mcgsm_samples[0]) > 0.0001)
self.assertTrue(ks_2samp(mogsm_samples[i], mcgsm_samples[1]) > 0.0001)
self.assertTrue(ks_2samp(mogsm_samples[i], mcgsm_samples[2]) > 0.0001)
self.assertTrue(ks_2samp(mogsm_samples[i], mcgsm_samples[0]).pvalue > 0.0001)
self.assertTrue(ks_2samp(mogsm_samples[i], mcgsm_samples[1]).pvalue > 0.0001)
self.assertTrue(ks_2samp(mogsm_samples[i], mcgsm_samples[2]).pvalue > 0.0001)

posterior = mcgsm.posterior(input, mcgsm_samples)

Expand Down Expand Up @@ -385,11 +385,11 @@ def test_pickle(self):
tmp_file = mkstemp()[1]

# store model
with open(tmp_file, 'w') as handle:
with open(tmp_file, 'wb') as handle:
dump({'mcgsm': mcgsm0}, handle)

# load model
with open(tmp_file) as handle:
with open(tmp_file, 'rb') as handle:
mcgsm1 = load(handle)['mcgsm']

# make sure parameters haven't changed
Expand Down Expand Up @@ -421,8 +421,8 @@ def test_patchmcgsm(self):

model = PatchMCGSM(8, 8, xmask, ymask, model=MCGSM(sum(xmask), 1))

self.assertLess(max(abs(model.input_mask() - xmask)), 1e-8)
self.assertLess(max(abs(model.output_mask() - ymask)), 1e-8)
self.assertLess(max(abs(model.input_mask() ^ xmask)), 1e-8)
self.assertLess(max(abs(model.output_mask() ^ ymask)), 1e-8)

for i in range(8):
for j in range(8):
Expand All @@ -435,8 +435,8 @@ def test_patchmcgsm(self):

model = PatchMCGSM(rows, cols, xmask, ymask, order, MCGSM(sum(xmask), 1))

self.assertLess(max(abs(model.input_mask() - xmask)), 1e-8)
self.assertLess(max(abs(model.output_mask() - ymask)), 1e-8)
self.assertLess(max(abs(model.input_mask() ^ xmask)), 1e-8)
self.assertLess(max(abs(model.output_mask() ^ ymask)), 1e-8)

for i in range(rows):
for j in range(cols):
Expand All @@ -446,8 +446,8 @@ def test_patchmcgsm(self):
model0 = PatchMCGSM(rows, cols, max_pcs=3)
model1 = PatchMCGSM(rows, cols, model0.input_mask(), model0.output_mask(), model0.order)

self.assertLess(max(abs(model0.input_mask() - model1.input_mask())), 1e-8)
self.assertLess(max(abs(model0.output_mask() - model1.output_mask())), 1e-8)
self.assertLess(max(abs(model0.input_mask() ^ model1.input_mask())), 1e-8)
self.assertLess(max(abs(model0.output_mask() ^ model1.output_mask())), 1e-8)
self.assertLess(max(abs(asarray(model0.order) - asarray(model1.order))), 1e-8)

# test computation of input masks
Expand All @@ -456,7 +456,7 @@ def test_patchmcgsm(self):
i, j = model0.order[0]
input_mask = model.input_mask(i, j)
for i, j in model.order[1:]:
self.assertEqual(sum(model.input_mask(i, j) - input_mask), 1)
self.assertEqual(sum(model.input_mask(i, j) ^ input_mask), 1)
input_mask = model.input_mask(i, j)


Expand Down
4 changes: 2 additions & 2 deletions code/cmt/python/tests/mixture_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,11 @@ def test_pickle(self):
tmp_file = mkstemp()[1]

# store model
with open(tmp_file, 'w') as handle:
with open(tmp_file, 'wb') as handle:
dump({'model': model0}, handle)

# load model
with open(tmp_file) as handle:
with open(tmp_file, 'rb') as handle:
model1 = load(handle)['model']

# make sure parameters haven't changed
Expand Down
4 changes: 2 additions & 2 deletions code/cmt/python/tests/mlr_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,11 +73,11 @@ def test_mlr_pickle(self):
model0.biases = randn(*model0.biases.shape)

# store model
with open(tmp_file, 'w') as handle:
with open(tmp_file, 'wb') as handle:
dump({'model': model0}, handle)

# load model
with open(tmp_file) as handle:
with open(tmp_file, 'rb') as handle:
model1 = load(handle)['model']

# make sure parameters haven't changed
Expand Down
Loading