Merge pull request #172 from kjd/optimize-contextj

More efficient resolution of joiner contexts

Author: kjd

idna/core.py

@@ -150,9 +150,11 @@ def valid_contextj(label: str, pos: int) -> bool:
             joining_type = idnadata.joining_types.get(ord(label[i]))
             if joining_type == ord('T'):
                 continue
-            if joining_type in [ord('L'), ord('D')]:
+            elif joining_type in [ord('L'), ord('D')]:
                 ok = True
                 break
+            else:
+                break
 
         if not ok:
             return False
@@ -162,9 +164,11 @@ def valid_contextj(label: str, pos: int) -> bool:
             joining_type = idnadata.joining_types.get(ord(label[i]))
             if joining_type == ord('T'):
                 continue
-            if joining_type in [ord('R'), ord('D')]:
+            elif joining_type in [ord('R'), ord('D')]:
                 ok = True
                 break
+            else:
+                break
         return ok
 
     if cp_value == 0x200d:
@@ -236,12 +240,8 @@ def check_label(label: Union[str, bytes, bytearray]) -> None:
         if intranges_contain(cp_value, idnadata.codepoint_classes['PVALID']):
             continue
         elif intranges_contain(cp_value, idnadata.codepoint_classes['CONTEXTJ']):
-            try:
-                if not valid_contextj(label, pos):
-                    raise InvalidCodepointContext('Joiner {} not allowed at position {} in {}'.format(
-                        _unot(cp_value), pos+1, repr(label)))
-            except ValueError:
-                raise IDNAError('Unknown codepoint adjacent to joiner {} at position {} in {}'.format(
+            if not valid_contextj(label, pos):
+                raise InvalidCodepointContext('Joiner {} not allowed at position {} in {}'.format(
                     _unot(cp_value), pos+1, repr(label)))
         elif intranges_contain(cp_value, idnadata.codepoint_classes['CONTEXTO']):
             if not valid_contexto(label, pos):

idna/idnadata.py

@@ -246,11 +246,18 @@ class UnicodeData(object):
     def _load_arabicshaping(self):
 
         self.ucd_as = {}
-        f_as = self._ucdfile('ArabicShaping.txt')
+        f_as = self._ucdfile('extracted/DerivedJoiningType.txt')
         for line in f_as.splitlines():
-            result = re.match(r'^(?P<cp>[0-9A-F]{4,6})\s*;\s*.*?\s*;\s*(?P<jt>\S+)\s*;', line)
+            result = re.match(
+                r'^(?P<start>[0-9A-F]{4,6})(|\.\.(?P<end>[0-9A-F]{4,6}))\s*;\s*(?P<jt>\S+)\s*(|\#.*)$',
+                line)
             if result:
-                self.ucd_as[int(result.group('cp'), 16)] = result.group('jt')
+                if result.group('end'):
+                    for i in hexrange(result.group('start'), result.group('end')):
+                        self.ucd_as[i] = result.group('jt')
+                else:
+                    i = hexvalue(result.group('start'))
+                    self.ucd_as[i] = result.group('jt')
 
     def _load_scripts(self):