Added valid_utf8(...) function.

Author: wcjohns

SpecUtils/StringAlgo.h

@@ -224,7 +224,11 @@ namespace  SpecUtils
   size_t utf8_str_size_limit( const char * const str,
                              size_t num_in_bytes, const size_t max_bytes );
 
-  
+  /** Returns if the passed-in string is valid UTF-8.
+   @param str The string to evaluate.
+   @param num_in_bytes The length of the string - must not include (optional) null-terminator.
+   */
+  bool valid_utf8( const char * const str, const size_t num_in_bytes );
 
 
   /** \brief parses a string of ascii characters to their floating point value.

src/StringAlgo.cpp

@@ -1090,6 +1090,42 @@ namespace SpecUtils
   }
 
 
+  bool valid_utf8( const char * const str, const size_t num_in_bytes )
+  {
+    int bytesToProcess = 0;
+    
+    for( size_t i = 0; i < num_in_bytes; ++i )
+    {
+      const uint8_t c = reinterpret_cast<const uint8_t &>( str[i] );
+      if( bytesToProcess == 0 )
+      {
+        // Determine how many bytes to expect
+        if( (c & 0x80) == 0 )
+          continue;            // 1-byte character (ASCII)
+        else if( (c & 0xE0) == 0xC0 )
+          bytesToProcess = 1;  // 2-byte character
+        else if( (c & 0xF0) == 0xE0 )
+          bytesToProcess = 2;  // 3-byte character
+        else if( (c & 0xF8) == 0xF0 )
+          bytesToProcess = 3;  // 4-byte character
+        else
+          return false; // Invalid leading byte
+      }else
+      {
+        // Expecting continuation byte
+        if( (c & 0xC0) != 0x80 )
+          return false; // Not a valid continuation byte
+        bytesToProcess--;
+        assert( bytesToProcess >= 0 );
+      }
+    }//for( size_t i = 0; i < num_in_bytes; ++i )
+    
+    assert( bytesToProcess >= 0 );
+    
+    return (bytesToProcess == 0);
+  }//bool valid_utf8( const char * const str, size_t num_in_bytes )
+  
+  
   template <class T>
   bool split_to_integral_types( const char *input, const size_t length,
                                std::vector<T> &results )

unit_tests/test_string_functions.cpp

@@ -823,3 +823,58 @@ TEST_CASE( "testPrintCompact" )
   check_range(-1000000.0,10000000.0);
   check_range(-1.0E32,1.0E32);
 }//void testPrintCompact()
+
+
+TEST_CASE( "testValidUtf8" )
+{
+  using namespace SpecUtils;
+  
+  // Valid UTF-8 strings
+  const char* valid_utf8_strings[] = {
+    "Hello, World!",               // Basic ASCII
+    "Привет",                       // Cyrillic
+    "こんにちは",                   // Japanese
+    "😊",                           // Emoji
+    "\xE2\x9C\x94",                // Check mark (U+2714)
+    "\xF0\x9F\x98\x81",            // Grinning face (U+1F600)
+    "\xE2\x82\xAC",                // Euro sign (U+20AC)
+    "\xF0\x9F\x8C\x90",            // Earth globe (U+1F30D)
+    "\xF0\x9F\x92\xA9",            // Money bag (U+1F4B0)
+  };
+  
+  for (const auto& str : valid_utf8_strings) {
+    CHECK(valid_utf8(str, std::strlen(str)));
+  }
+  
+  // Invalid UTF-8 strings
+  const char* invalid_utf8_strings[] = {
+    "\x80",                         // Invalid start byte
+    "\xC3\x28",                     // Invalid continuation byte
+    "\xE2\x82\x28",                 // Invalid continuation byte
+    "\xF0\x28\x8C\x28",             // Invalid continuation byte
+    "\xF0\x9F\x98\x28",             // Invalid continuation byte
+    "\xC3\xA9\xC3\x28",             // Mixed valid and invalid
+    "\xE2\x82\xAC\xE2\x28",         // Mixed valid and invalid
+    "\xF0\x9F\x92\xA9\xF0\x28",     // Mixed valid and invalid
+  };
+  
+  for (const auto& str : invalid_utf8_strings) {
+    CHECK(!valid_utf8(str, std::strlen(str)));
+  }
+  
+  
+  // Edge cases
+  CHECK(valid_utf8("", 0)); // Empty string
+  CHECK(valid_utf8("\xC2\xA9", 2)); // Single valid UTF-8 character (©)
+  CHECK(!valid_utf8("\xC2", 1)); // Incomplete multibyte sequence
+  CHECK(!valid_utf8("\xE2\x82", 2)); // Incomplete multibyte sequence
+  CHECK(!valid_utf8("\xF0\x9F\x98", 3)); // Incomplete multibyte sequence
+  
+  // Large valid UTF-8 string
+  std::string large_valid_utf8(10000, 'a'); // A large string of 'a's
+  CHECK(valid_utf8(large_valid_utf8.c_str(), large_valid_utf8.size()));
+  
+  // Large invalid UTF-8 string
+  std::string large_invalid_utf8(10000, '\x80'); // A large string of invalid bytes
+  CHECK(!valid_utf8(large_invalid_utf8.c_str(), large_invalid_utf8.size()));
+}//TEST_CASE( "testValidUtf8" )