Mop-up for commit 85feb77.

Adjust commentary in regc_pg_locale.c to remove mention of the possibility
of not having <wctype.h> functions, since we no longer consider that.

Eliminate duplicate code in wparser_def.c by generalizing the p_iswhat
macro to take a parameter saying what to return for non-ASCII chars
in C locale.  (That's not really a consequence of the
USE_WIDE_UPPER_LOWER-ectomy, but I noticed it while doing that.)

Author: za-arthur

Diff for: expected/pg_tsparser.out

@@ -236,3 +236,31 @@ SELECT to_tsvector('english_ts', 'test2.com');
  'com':3 'test2':2 'test2.com':1
 (1 row)
 
+-- Test non-ASCII symbols
+SELECT * from ts_parse('tsparser', 'аб_вгд 12_абв 12-абв абв.рф абв2.рф');
+ tokid | token  
+-------+--------
+    17 | аб_вгд
+    10 | аб
+    12 | _
+    10 | вгд
+    12 |  
+    15 | 12_абв
+     9 | 12
+    12 | _
+    10 | абв
+    12 |  
+    15 | 12-абв
+     9 | 12
+    12 | -
+    10 | абв
+    12 |  
+     2 | абв
+    12 | .
+     2 | рф
+    12 |  
+     3 | абв2
+    12 | .
+     2 | рф
+(22 rows)
+

Diff for: sql/pg_tsparser.sql

@@ -26,3 +26,6 @@ SELECT to_tsvector('english_ts', '12_abc');
 SELECT to_tsvector('english_ts', '12-abc');
 SELECT to_tsvector('english_ts', 'test.com');
 SELECT to_tsvector('english_ts', 'test2.com');
+
+-- Test non-ASCII symbols
+SELECT * from ts_parse('tsparser', 'аб_вгд 12_абв 12-абв абв.рф абв2.рф');

Diff for: tsparser.c

@@ -249,11 +249,9 @@ typedef struct TParser
 	/* string and position information */
 	char	   *str;			/* multibyte string */
 	int			lenstr;			/* length of mbstring */
-#ifdef USE_WIDE_UPPER_LOWER
 	wchar_t    *wstr;			/* wide character string */
 	pg_wchar   *pgwstr;			/* wide character string for C-locale */
 	bool		usewide;
-#endif
 
 	/* State of parse */
 	int			charmaxlen;
@@ -302,8 +300,6 @@ TParserInit(char *str, int len)
 	prs->str = str;
 	prs->lenstr = len;
 
-#ifdef USE_WIDE_UPPER_LOWER
-
 	/*
 	 * Use wide char code only when max encoding length > 1.
 	 */
@@ -331,7 +327,6 @@ TParserInit(char *str, int len)
 	}
 	else
 		prs->usewide = false;
-#endif
 
 	prs->state = newTParserPosition(NULL);
 	prs->state->state = TPS_Base;
@@ -368,15 +363,12 @@ TParserCopyInit(const TParser *orig)
 	prs->charmaxlen = orig->charmaxlen;
 	prs->str = orig->str + orig->state->posbyte;
 	prs->lenstr = orig->lenstr - orig->state->posbyte;
-
-#ifdef USE_WIDE_UPPER_LOWER
 	prs->usewide = orig->usewide;
 
 	if (orig->pgwstr)
 		prs->pgwstr = orig->pgwstr + orig->state->poschar;
 	if (orig->wstr)
 		prs->wstr = orig->wstr + orig->state->poschar;
-#endif
 
 	prs->state = newTParserPosition(NULL);
 	prs->state->state = TPS_Base;
@@ -401,12 +393,10 @@ TParserClose(TParser *prs)
 		prs->state = ptr;
 	}
 
-#ifdef USE_WIDE_UPPER_LOWER
 	if (prs->wstr)
 		pfree(prs->wstr);
 	if (prs->pgwstr)
 		pfree(prs->pgwstr);
-#endif
 
 #ifdef WPARSER_TRACE
 	fprintf(stderr, "closing parser\n");
@@ -445,96 +435,45 @@ TParserCopyClose(TParser *prs)
  *	- if locale is C then we use pgwstr instead of wstr.
  */
 
-#ifdef USE_WIDE_UPPER_LOWER
-
-#define p_iswhat(type)														\
+#define p_iswhat(type, nonascii)											\
+																			\
 static int																	\
-p_is##type(TParser *prs) {													\
-	Assert( prs->state );													\
-	if ( prs->usewide )														\
+p_is##type(TParser *prs)													\
+{																			\
+	Assert(prs->state);														\
+	if (prs->usewide)														\
 	{																		\
-		if ( prs->pgwstr )													\
+		if (prs->pgwstr)													\
 		{																	\
 			unsigned int c = *(prs->pgwstr + prs->state->poschar);			\
-			if ( c > 0x7f )													\
-				return 0;													\
-			return is##type( c );											\
+			if (c > 0x7f)													\
+				return nonascii;											\
+			return is##type(c);												\
 		}																	\
-		return isw##type( *( prs->wstr + prs->state->poschar ) );			\
+		return isw##type(*(prs->wstr + prs->state->poschar));				\
 	}																		\
-																			\
-	return is##type( *(unsigned char*)( prs->str + prs->state->posbyte ) ); \
-}	\
+	return is##type(*(unsigned char *) (prs->str + prs->state->posbyte));	\
+}																			\
 																			\
 static int																	\
-p_isnot##type(TParser *prs) {												\
+p_isnot##type(TParser *prs)													\
+{																			\
 	return !p_is##type(prs);												\
 }
 
-static int
-p_isalnum(TParser *prs)
-{
-	Assert(prs->state);
-
-	if (prs->usewide)
-	{
-		if (prs->pgwstr)
-		{
-			unsigned int c = *(prs->pgwstr + prs->state->poschar);
-
-			/*
-			 * any non-ascii symbol with multibyte encoding with C-locale is
-			 * an alpha character
-			 */
-			if (c > 0x7f)
-				return 1;
-
-			return isalnum(c);
-		}
-
-		return iswalnum(*(prs->wstr + prs->state->poschar));
-	}
-
-	return isalnum(*(unsigned char *) (prs->str + prs->state->posbyte));
-}
-static int
-p_isnotalnum(TParser *prs)
-{
-	return !p_isalnum(prs);
-}
-
-static int
-p_isalpha(TParser *prs)
-{
-	Assert(prs->state);
-
-	if (prs->usewide)
-	{
-		if (prs->pgwstr)
-		{
-			unsigned int c = *(prs->pgwstr + prs->state->poschar);
-
-			/*
-			 * any non-ascii symbol with multibyte encoding with C-locale is
-			 * an alpha character
-			 */
-			if (c > 0x7f)
-				return 1;
-
-			return isalpha(c);
-		}
-
-		return iswalpha(*(prs->wstr + prs->state->poschar));
-	}
-
-	return isalpha(*(unsigned char *) (prs->str + prs->state->posbyte));
-}
-
-static int
-p_isnotalpha(TParser *prs)
-{
-	return !p_isalpha(prs);
-}
+/*
+ * In C locale with a multibyte encoding, any non-ASCII symbol is considered
+ * an alpha character, but not a member of other char classes.
+ */
+p_iswhat(alnum, 1)
+p_iswhat(alpha, 1)
+p_iswhat(digit, 0)
+p_iswhat(lower, 0)
+p_iswhat(print, 0)
+p_iswhat(punct, 0)
+p_iswhat(space, 0)
+p_iswhat(upper, 0)
+p_iswhat(xdigit, 0)
 
 /* p_iseq should be used only for ascii symbols */
 
@@ -544,39 +483,6 @@ p_iseq(TParser *prs, char c)
 	Assert(prs->state);
 	return ((prs->state->charlen == 1 && *(prs->str + prs->state->posbyte) == c)) ? 1 : 0;
 }
-#else							/* USE_WIDE_UPPER_LOWER */
-
-#define p_iswhat(type)														\
-static int																	\
-p_is##type(TParser *prs) {													\
-	Assert( prs->state );													\
-	return is##type( (unsigned char)*( prs->str + prs->state->posbyte ) );	\
-}	\
-																			\
-static int																	\
-p_isnot##type(TParser *prs) {												\
-	return !p_is##type(prs);												\
-}
-
-
-static int
-p_iseq(TParser *prs, char c)
-{
-	Assert(prs->state);
-	return (*(prs->str + prs->state->posbyte) == c) ? 1 : 0;
-}
-
-p_iswhat(alnum)
-p_iswhat(alpha)
-#endif   /* USE_WIDE_UPPER_LOWER */
-
-p_iswhat(digit)
-p_iswhat(lower)
-p_iswhat(print)
-p_iswhat(punct)
-p_iswhat(space)
-p_iswhat(upper)
-p_iswhat(xdigit)
 
 static int
 p_isEOF(TParser *prs)
@@ -793,8 +699,6 @@ p_isspecial(TParser *prs)
 	if (pg_dsplen(prs->str + prs->state->posbyte) == 0)
 		return 1;
 
-#ifdef USE_WIDE_UPPER_LOWER
-
 	/*
 	 * Unicode Characters in the 'Mark, Spacing Combining' Category That
 	 * characters are not alpha although they are not breakers of word too.
@@ -1058,7 +962,6 @@ p_isspecial(TParser *prs)
 				StopHigh = StopMiddle;
 		}
 	}
-#endif
 
 	return 0;
 }