Merge branch 'release/0.7.2'

Author: emfomy

ckipnlp/__init__.py

@@ -6,7 +6,7 @@
 __copyright__ = '2018-2020 CKIP Lab'
 
 __title__ = 'CKIPNLP'
-__version__ = '0.7.1'
+__version__ = '0.7.2'
 __description__ = 'CKIP CoreNLP Wrappers'
 __license__ = 'CC BY-NC-SA 4.0'
 

ckipnlp/parser/__init__.py

@@ -91,16 +91,35 @@ def CkipParser(*, _=None): pass # pylint: disable=redefined-outer-name, invalid-
         except: # pylint: disable=bare-except
             pass
 
+    @staticmethod
+    def normalize_text(text):
+        """Text normalization output.
+
+        Replacing keywords ``()+-:|&#`` by by full-width ones.
+        """
+        return (text
+            .replace('(', '（')
+            .replace(')', '）')
+            .replace('+', '＋')
+            .replace('-', '－')
+            .replace(':', '：')
+            .replace('|', '｜')
+            .replace('&', '＆') # for tree draw
+            .replace('#', '＃') # for tree draw
+        )
+
     def __call__(self, text):
         return self.apply(text)
 
-    def apply(self, text):
+    def apply(self, text, *, normalize=True):
         """Parse a sentence.
 
         Parameters
         ----------
             text : str
                 the input sentence.
+            normalize : bool
+                do text normalization (please refer :meth:`normalize_text`).
 
         Return
         ------
@@ -110,21 +129,25 @@ def apply(self, text):
         .. hint::
             One may also call this method as :meth:`__call__`.
         """
-        return self.apply_list([text])[0]
+        return self.apply_list([text], normalize=normalize)[0]
 
-    def apply_list(self, ilist):
+    def apply_list(self, ilist, *, normalize=True):
         """Parse a list of sentences.
 
         Parameters
         ----------
             ilist : List[str]
                 the list of input sentences.
+            normalize : bool
+                do text normalization (please refer :meth:`normalize_text`).
 
         Return
         ------
             List[str]
                 the list of output sentences.
         """
+        if normalize:
+            ilist = list(map(self.normalize_text, ilist))
         return self.__core.apply_list(ilist)
 
     def apply_file(self, ifile, ofile):

ckipnlp/ws/__init__.py

@@ -78,16 +78,29 @@ def CkipWs(*, _=None): pass # pylint: disable=redefined-outer-name, invalid-name
         except: # pylint: disable=bare-except
             pass
 
+    @staticmethod
+    def normalize_text(text):
+        """Text normalization output.
+
+        Replacing keywords ``()+-:|&#`` by by full-width ones.
+        """
+        return (text
+            .replace('(', '（')
+            .replace(')', '）') # for tree draw
+        )
+
     def __call__(self, text):
         return self.apply(text)
 
-    def apply(self, text):
-        """Segment a sentence.
+    def apply(self, text, *, normalize=True):
+        """Parse a sentence.
 
         Parameters
         ----------
             text : str
                 the input sentence.
+            normalize : bool
+                do text normalization (please refer :meth:`normalize_text`).
 
         Return
         ------
@@ -97,21 +110,25 @@ def apply(self, text):
         .. hint::
             One may also call this method as :meth:`__call__`.
         """
-        return self.apply_list([text])[0]
+        return self.apply_list([text], normalize=normalize)[0]
 
-    def apply_list(self, ilist):
-        """Segment a list of sentences.
+    def apply_list(self, ilist, *, normalize=True):
+        """Parse a list of sentences.
 
         Parameters
         ----------
             ilist : List[str]
                 the list of input sentences.
+            normalize : bool
+                do text normalization (please refer :meth:`normalize_text`).
 
         Return
         ------
             List[str]
                 the list of output sentences.
         """
+        if normalize:
+            ilist = list(map(self.normalize_text, ilist))
         return self.__core.apply_list(ilist)
 
     def apply_file(self, ifile, ofile, uwfile=''):

ckipparser/__init__.py

@@ -32,8 +32,67 @@
 
 import ckipnlp as about
 
-with open('README.rst', encoding='utf-8') as fin:
-    readme = fin.read()
+################################################################################
+
+def main():
+
+    with open('README.rst', encoding='utf-8') as fin:
+        readme = fin.read()
+
+    setup(
+        name=about.__name__,
+        version=about.__version__,
+        author=about.__author_name__,
+        author_email=about.__author_email__,
+        description=about.__description__,
+        long_description=readme,
+        long_description_content_type='text/x-rst',
+        url=about.__url__,
+        download_url=about.__download_url__,
+        platforms=['linux_x86_64'],
+        license=about.__license__,
+        classifiers=[
+            'Development Status :: 4 - Beta',
+            'Environment :: Console',
+            'Programming Language :: Python',
+            'Programming Language :: Python :: 3',
+            'Programming Language :: Python :: 3.5',
+            'Programming Language :: Python :: 3.6',
+            'Programming Language :: Python :: 3.7',
+            'Programming Language :: Python :: 3.8',
+            'Programming Language :: Python :: 3.9',
+            'Programming Language :: Python :: 3 :: Only',
+            'Programming Language :: Cython',
+            'License :: Free for non-commercial use',
+            'Operating System :: POSIX :: Linux',
+            'Natural Language :: Chinese (Traditional)',
+        ],
+        python_requires='>=3.5',
+        packages=find_namespace_packages(include=['ckipnlp', 'ckipnlp.*',]),
+        install_requires=[
+            'treelib>=1.5.5',
+        ],
+        ext_modules=cythonize(
+            [
+                Extension('ckipnlp._core.ws',
+                    sources=['src/ws/ckipws.pyx'],
+                    libraries=['WordSeg'],
+                    language='c++',
+                ),
+                Extension('ckipnlp._core.parser',
+                    sources=['src/parser/ckipparser.pyx'],
+                    libraries=['CKIPCoreNLP', 'CKIPParser', 'CKIPWS', 'CKIPSRL'],
+                    language='c++',
+                ),
+            ],
+            build_dir='build',
+        ),
+        data_files=[],
+        cmdclass={
+            'install': InstallCommand,
+            'develop': DevelopCommand,
+        },
+    )
 
 ################################################################################
 
@@ -97,10 +156,10 @@ def finalize_options(self):
 
         # subdirectory
         opt_subdirectory = [
-            ('ws_lib_dir',      'ws_dir',     'lib',),
-            ('ws_share_dir',    'ws_dir',     '',),
-            ('parser_lib_dir',  'parser_dir', 'lib',),
-            ('parser_share_dir','parser_dir', '',),
+            ('ws_lib_dir',       'ws_dir',     'lib',),
+            ('ws_share_dir',     'ws_dir',     '',),
+            ('parser_lib_dir',   'parser_dir', 'lib',),
+            ('parser_share_dir', 'parser_dir', '',),
 
             ('data2_dir', 'ws_share_dir',     'Data2',),
             ('data2_dir', 'parser_share_dir', 'Data2',),
@@ -199,52 +258,5 @@ def __init__(self, *args, **kwargs):
 
 ################################################################################
 
-setup(
-    name=about.__name__,
-    version=about.__version__,
-    author=about.__author_name__,
-    author_email=about.__author_email__,
-    description=about.__description__,
-    long_description=readme,
-    long_description_content_type='text/x-rst',
-    url=about.__url__,
-    download_url=about.__download_url__,
-    platforms=['linux_x86_64'],
-    license=about.__license__,
-    classifiers=[
-        'Development Status :: 4 - Beta',
-        'Environment :: Console',
-        'Programming Language :: Python',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Cython',
-        'License :: Free for non-commercial use',
-        'Operating System :: POSIX :: Linux',
-        'Natural Language :: Chinese (Traditional)',
-    ],
-    python_requires='>=3.5',
-    packages=find_namespace_packages(include=['ckipnlp', 'ckipnlp.*', 'ckipws', 'ckipparser',]),
-    install_requires=[
-        'treelib>=1.5.5',
-    ],
-    ext_modules=cythonize(
-        [
-            Extension('ckipnlp._core.ws',
-                sources=['src/ws/ckipws.pyx'],
-                libraries=['WordSeg'],
-            ),
-            Extension('ckipnlp._core.parser',
-                sources=['src/parser/ckipparser.pyx'],
-                libraries=['CKIPCoreNLP','CKIPParser','CKIPWS','CKIPSRL'],
-            ),
-        ],
-        build_dir='build',
-    ),
-    data_files=[],
-    cmdclass={
-        'install': InstallCommand,
-        'develop': DevelopCommand,
-    },
-)
+if __name__ == '__main__':
+    main()

ckipws/__init__.py

@@ -6,14 +6,14 @@ __license__ = 'CC BY-NC-SA 4.0'
 
 cdef extern:
 
-	ctypedef void* corenlp_t
+	ctypedef void* corenlp_t;
 
-	corenlp_t CKIPCoreNLP_New()
-	int CKIPCoreNLP_InitData(corenlp_t obj, char *FileName);
-	int CKIPCoreNLP_ApplyFile(corenlp_t obj, char *input, char *output);
-	int CKIPCoreNLP_ApplyList(corenlp_t obj, int length, const Py_UNICODE **inputList);
-	int CKIPCoreNLP_Parse(corenlp_t obj, const Py_UNICODE* pwsText, Py_UNICODE** ppwsResult);
-	int CKIPCoreNLP_ParseFile(corenlp_t obj, char *input, char *output);
-	const Py_UNICODE* CKIPCoreNLP_GetResultBegin(corenlp_t obj);
-	const Py_UNICODE* CKIPCoreNLP_GetResultNext(corenlp_t obj);
-	void CKIPCoreNLP_Destroy(corenlp_t obj);
+	corenlp_t CKIPCoreNLP_New() nogil;
+	int CKIPCoreNLP_InitData(corenlp_t obj, char *FileName) nogil;
+	int CKIPCoreNLP_ApplyFile(corenlp_t obj, char *input, char *output) nogil;
+	int CKIPCoreNLP_ApplyList(corenlp_t obj, int length, const Py_UNICODE **inputList) nogil;
+	int CKIPCoreNLP_Parse(corenlp_t obj, const Py_UNICODE* pwsText, Py_UNICODE** ppwsResult) nogil;
+	int CKIPCoreNLP_ParseFile(corenlp_t obj, char *input, char *output) nogil;
+	const Py_UNICODE* CKIPCoreNLP_GetResultBegin(corenlp_t obj) nogil;
+	const Py_UNICODE* CKIPCoreNLP_GetResultNext(corenlp_t obj) nogil;
+	void CKIPCoreNLP_Destroy(corenlp_t obj) nogil;

setup.py

@@ -6,9 +6,10 @@ __copyright__ = '2018-2020 CKIP Lab'
 __license__ = 'CC BY-NC-SA 4.0'
 
 cimport src.parser.cckipparser as cckipparser
-from libc.stdlib cimport malloc, free
-from cpython.unicode cimport PyUnicode_AsUnicode
+cimport cython
+from libcpp.vector cimport vector
 
+@cython.final
 cdef class CkipParserCore:
 
     cdef cckipparser.corenlp_t __obj
@@ -19,31 +20,28 @@ cdef class CkipParserCore:
     def __dealloc__(self):
         if self.__obj is not NULL:
             cckipparser.CKIPCoreNLP_Destroy(self.__obj)
-            pass
 
-    def init_data(self, inifile):
+    def init_data(self, str inifile):
         ret = cckipparser.CKIPCoreNLP_InitData(self.__obj, inifile.encode())
         if not ret:
             raise IOError()
 
-    def apply_list(self, ilist):
-        inum = len(ilist)
+    # def enable_logger(self):
+    #     cckipparser.CKIPCoreNLP_EnableConsoleLogger(self.__obj)
 
-        iarr = <const Py_UNICODE**> malloc(sizeof(const Py_UNICODE*) * inum)
-        for i in range(inum):
-            iarr[i] = PyUnicode_AsUnicode(ilist[i])
-        ret = cckipparser.CKIPCoreNLP_ApplyList(self.__obj, inum, iarr)
-        free(iarr)
+    def apply_list(self, vector[const Py_UNICODE*] ilist):
+
+        ret = cckipparser.CKIPCoreNLP_ApplyList(self.__obj, ilist.size(), ilist.data())
         assert ret is not None
 
-        olist = []
+        cdef vector[const Py_UNICODE*] olist
         result = cckipparser.CKIPCoreNLP_GetResultBegin(self.__obj)
         while result is not NULL:
-            olist.append(result.strip())
+            olist.push_back(result)
             result = cckipparser.CKIPCoreNLP_GetResultNext(self.__obj)
 
         return olist
 
-    def apply_file(self, ifile, ofile):
+    def apply_file(self, str ifile, str ofile):
         ret = cckipparser.CKIPCoreNLP_ApplyFile(self.__obj, ifile.encode(), ofile.encode())
         assert ret is not None