Fix some minor warnings/issues.

Also exposed some string utils for exporting spectra to HTML/JS, to the fuzzing routine.

Author: wcjohns

SpecUtils/D3SpectrumExport.h

@@ -24,13 +24,17 @@
 #ifndef D3SpectrumExport_h
 #define D3SpectrumExport_h
 
+#include "SpecUtils_config.h"
+
 #include <map>
 #include <vector>
 #include <string>
 #include <utility>
 #include <ostream>
 
-#include "SpecUtils_config.h"
+#if( SpecUtils_BUILD_FUZZING_TESTS )
+#include <sstream>
+#endif
 
 
 namespace SpecUtils
@@ -48,7 +52,7 @@ namespace D3SpectrumExport
   struct D3SpectrumChartOptions;
 
   //Writes the spectrum component of the js that SpectrumChartD3 expects.
-  // An anotatated example of what would be written is:
+  // An annotated example of what would be written is:
   // { "title": "Line Title", "peaks":"[...]", "liveTime": 300, "realTime": 320, "neutrons": 0,
   //   "lineColor": "black", "x": [0, 2.93, 5.86, ..., 3000], "y": [0,1,10,...0], "yScaleFactor": 1.0 }
   bool write_spectrum_data_js( std::ostream &ostr,
@@ -188,5 +192,11 @@ namespace D3SpectrumExport
 
     std::map<std::string,std::string> m_reference_lines_json;  //map from nuclide to their JSON
   };//struct D3SpectrumChartOptions
+  
+#if( SpecUtils_BUILD_FUZZING_TESTS )
+  std::string escape_text_test( const std::string &input );
+  void copy_check_utf8_test( const char * &src, char * &dest );
+  void sanitize_unicode_test( std::stringstream &sout, const std::string& text );
+#endif
 }//namespace D3SpectrumExport
 #endif //D3SpectrumExport

SpecUtils/StringAlgo.h

@@ -314,7 +314,7 @@ namespace  SpecUtils
   bool split_to_floats( const char *input, const size_t length,
                        std::vector<float> &results );
 
-  /* \brief A convienience function. */
+  /* \brief A convenience function. */
   bool split_to_floats( const std::string &input, std::vector<float> &results );
 
 

fuzz_test/fuzz_str_utils.cpp

@@ -27,11 +27,15 @@
 
 #include <random>
 
+#include "SpecUtils/DateTime.h"
 #include "SpecUtils/StringAlgo.h"
 #include "SpecUtils/ParseUtils.h"
 #include "SpecUtils/Filesystem.h"
-#include "SpecUtils/DateTime.h"
-
+#if( SpecUtils_ENABLE_D3_CHART )
+#include "SpecUtils/D3SpectrumExport.h"
+#else
+static_assert( 0, "You should enable SpecUtils_ENABLE_D3_CHART" );
+#endif
 
 using namespace std;
 
@@ -692,6 +696,21 @@ extern "C" int LLVMFuzzerTestOneInput( const uint8_t *data, size_t size )
   //std::istream &read_binary_data( std::istream &input, T &val );
   //size_t write_binary_data( std::ostream &input, const T &val );
 
+  #if( SpecUtils_ENABLE_D3_CHART )
+  {
+    D3SpectrumExport::escape_text_test( datastr );
+
+    char buf[4];
+    for( const char *c = datastr.c_str(); *c; ) 
+    {
+      char *b = buf;
+      D3SpectrumExport::copy_check_utf8_test(c, b);
+    }
+
+    std::stringstream outstrm;
+    D3SpectrumExport::sanitize_unicode_test( outstrm, datastr );
+  }
+  #endif
 
   return 0;
 }//LLVMFuzzerTestOneInput

src/D3SpectrumExport.cpp

@@ -19,6 +19,8 @@
 
 #include "SpecUtils_config.h"
 
+#include <iostream>
+
 #include <cmath>
 #include <stack>
 #include <limits>
@@ -45,8 +47,8 @@ using namespace std;
 
 namespace
 {
-  //anaonomous namespace for functions to help parse D3.js HTML files, that wont be
-  //  usefull outside of this file
+  //anonymous namespace for functions to help parse D3.js HTML files, that wont be
+  //  useful outside of this file
 #if( SpecUtils_D3_SUPPORT_FILE_STATIC )
   const unsigned char * const ns_libJsFiles[] = {
     D3_MIN_JS // D3.js library
@@ -82,19 +84,31 @@ namespace
 
   //Taken from the rapidxml.hpp that Wt uses
   template <class Ch>
-  void copy_check_utf8(const Ch *& src, Ch *& dest)
+  void copy_check_utf8( const Ch *& src, Ch *& dest )
   {
     // skip entire UTF-8 encoded characters at once,
     // checking their validity based on
     // http://www.dwheeler.com/secure-programs/Secure-Programs-HOWTO/character-encoding.html (5.9.4 column 3)
 
+    assert( src );
+    size_t src_len;
+    for( src_len = 0; src[src_len] && src_len < 4; ++src_len )
+    {
+    }
+    assert( src_len );
+    if( !src_len )
+      return;
+    
+    
     unsigned length = 1;
     bool legal = false;
-    if ((unsigned char)src[0] <= 0x7F) {
+    if ((unsigned char)src[0] <= 0x7F)
+    {
       unsigned char c = src[0];
       if (c == 0x09 || c == 0x0A || c == 0x0D || c >= 0x20)
         legal = true;
-    } else if ((unsigned char)src[0] >= 0xF0) {
+    }else if( ((unsigned char)src[0] >= 0xF0) && (src_len >= 4) )
+    {
       length = 4;
 
       if ((
@@ -120,7 +134,8 @@ namespace
            ))
         legal = true;
 
-    } else if ((unsigned char)src[0] >= 0xE0) {
+    }else if( ((unsigned char)src[0] >= 0xE0) && (src_len >= 3) )
+    {
       length = 3;
 
       if ((
@@ -142,7 +157,8 @@ namespace
            ))
         legal = true;
 
-    } else if ((unsigned char)src[0] >= 0xC0) {
+    }else if( ((unsigned char)src[0] >= 0xC0) && (src_len >= 2) )
+    {
       length = 2;
 
       if (
@@ -155,9 +171,12 @@ namespace
         legal = true;
     }
 
-    if (legal) {
-      if (dest) {
-        if (length == 3) {
+    if( legal )
+    {
+      if( dest )
+      {
+        if( length == 3 )
+        {
           /*
            U+2028 and U+2029 may cause problems, they are line
            separators that mess up JavaScript string literals.
@@ -172,44 +191,56 @@ namespace
               } else
                 for (unsigned i = 0; i < length; ++i)
                   *dest++ = *src++;
-        } else
-          for (unsigned i = 0; i < length; ++i)
+        }else
+        {
+          for( unsigned i = 0; i < length; ++i )
             *dest++ = *src++;
-      } else
+        }
+      }else
+      {
         src += length;
-    } else {
-      if (dest)
-        if (length >= 3) {
+      }
+    }else  //if( legal )
+    {
+      if( dest )
+      {
+        if( length >= 3 )
+        { 
           /* insert U+FFFD, the replacement character */
           *dest++ = (Ch)0xef;
           *dest++ = (Ch)0xbf;
           *dest++ = (Ch)0xbd;
           src += length;
-        } else
-          for (unsigned i = 0; i < length; ++i) {
+        }else
+        {
+          for (unsigned i = 0; i < length; ++i)
+          {
             *dest++ = '?';
             src++;
           }
-        else {
-          //const Ch *problem_src = src;
-          src += length;
-          throw runtime_error( "Invalid UTF-8 sequence" /* + const_cast<Ch *>(problem_src)*/ );
         }
-    }
-  }
-  
-
+      }else
+      {
+        //const Ch *problem_src = src;
+        src += length;
+        throw runtime_error( "Invalid UTF-8 sequence" /* + const_cast<Ch *>(problem_src)*/ );
+      }//if( dest )
+    }//if( legal ) / else
+  }//void copy_check_utf8
 
   void sanitize_unicode( stringstream &sout, const std::string& text )
   {
     char buf[4];
 
     for (const char *c = text.c_str(); *c;) {
+      assert( c <= (text.c_str() + text.size()) );
       char *b = buf;
       // but copy_check_utf8() does not declare the following ranges illegal:
       //  U+D800-U+DFFF
       //  U+FFFE-U+FFFF
       copy_check_utf8<char>(c, b);
+      assert( c <= (text.c_str() + text.size()) );
+      assert( (b - buf) <= 4 );
       for (char *i = buf; i < b; ++i)
         sout << *i;
     }
@@ -264,7 +295,24 @@ namespace
 
 namespace D3SpectrumExport
 {
-
+#if( SpecUtils_BUILD_FUZZING_TESTS )
+  std::string escape_text_test( const std::string &input )
+  {
+    return escape_text( input );
+  }
+  
+  void copy_check_utf8_test( const char *& src, char *& dest )
+  {
+    copy_check_utf8( src, dest );
+  }
+  
+  void sanitize_unicode_test( std::stringstream &sout, const std::string& text )
+  {
+    sanitize_unicode( sout, text );
+  }
+#endif
+  
+  
 #if( SpecUtils_D3_SUPPORT_FILE_STATIC )
   const unsigned char *d3_js(){ return D3_MIN_JS; }
   const unsigned char *spectrum_chart_d3_js(){ return SPECTRUM_CHART_D3_JS; }

src/SpecFile_csv.cpp

@@ -409,7 +409,7 @@ bool SpecFile::load_from_D3S_raw( std::istream &input )
 
         // This next call will check that energies are monotonically increasing, and if not throw
         //  an exception.
-        energy_cal->set_lower_channel_energy( channel_energies.size(), move(channel_energies) );
+        energy_cal->set_lower_channel_energy( channel_energies.size(), std::move(channel_energies) );
       }catch(std::exception &e)
       {
         energy_cal.reset();

src/SpecFile_gr135.cpp

@@ -1020,7 +1020,7 @@ bool SpecFile::write_binary_exploranium_gr135v2( std::ostream &output ) const
 
       memcpy( buffer + 19, &noutchannel, sizeof(noutchannel) );
 
-      double rt_f = std::round( 1000 * meas.real_time_ );
+      double rt_f = std::round( 1000 * std::max(meas.real_time_, 0.0f) );
       rt_f = std::min( rt_f, static_cast<double>(std::numeric_limits<uint32_t>::max()) );
 
       const uint32_t real_time_thousanths = static_cast<uint32_t>( rt_f );

src/SpecFile_n42.cpp

@@ -1163,7 +1163,7 @@ void add_spectra_to_measurement_node_in_2012_N42_xml( ::rapidxml::xml_node<char>
 
       string channeldata;
       if( !zerocompressed )
-        channeldata.reserve( 3*m->gamma_counts()->size() ); //3 has not been verified to be reasonalbe
+        channeldata.reserve( 3*m->gamma_counts()->size() ); //3 has not been verified to be reasonable
 
       const size_t nchannel = data.size();
 

src/SpecFile_spe.cpp

@@ -364,15 +364,13 @@ bool SpecFile::load_from_iaea( std::istream& istr )
         vector<string> channelstrs;
         split( channelstrs, line, " \t," );
 
-        unsigned int firstchannel = 0, lastchannel = 0;
+        int firstchannel = 0, lastchannel = 0;
         if( channelstrs.size() == 2 )
         {
-          try
-          {
-            firstchannel = atol( channelstrs[0].c_str() );
-            lastchannel = atol( channelstrs[1].c_str() );
-          }catch(...)
+          if( !parse_int(channelstrs[0].c_str(), channelstrs[0].size(), firstchannel)
+              || !parse_int(channelstrs[1].c_str(), channelstrs[1].size(), lastchannel) )
           {
+            firstchannel = lastchannel = 0;
           }
         }else
         {
@@ -382,11 +380,14 @@ bool SpecFile::load_from_iaea( std::istream& istr )
 
         double sum = 0.0;
         std::shared_ptr< vector<float> > channel_data( new vector<float>() );
-        if( firstchannel < lastchannel )
-          channel_data->reserve( lastchannel - firstchannel + 1 );
+        if( (firstchannel < lastchannel)
+           && (firstchannel >= 0)
+           && ((lastchannel - firstchannel) < (65536 + 2)) )
+        {
+          channel_data->reserve( static_cast<size_t>(lastchannel - firstchannel) + 1 );
+        }
 
-        //XXX - for some reason I think this next test condition is a little
-        //      fragile...
+        //TODO: for some reason I think this next test condition is a little fragile...
         int num_cd_error = 0, num_cd_error_current = 0;
         while( SpecUtils::safe_get_line( istr, line ) )
         {