PP: Clean up and rationalize floating-point-number scanner.

This encapsulates where the string could overflow, removing 40 lines
of fragile code.  It also improves handling of numbers that are too long.

There are a couple of open issues that could related to this function
being more rational (locale dependence, 1.#INF).

Test/baseResults/tokenLength.vert.out

@@ -11,8 +11,6 @@ ERROR: 0:34: '' : octal literal too big
 ERROR: 0:35: '' : numeric literal too long 
 ERROR: 0:35: '' : numeric literal too big 
 ERROR: 0:36: '' : float literal too long 
-ERROR: 0:36: '' : float literal too long 
-ERROR: 0:36: '' : float literal too long 
 WARNING: 0:39: '#extension' : extension not supported: a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhooooooooooooooooooooooooooooooohhhhhhhhhhhhhhhhh01234
 ERROR: 0:40: '' : name too long 
 WARNING: 0:40: '#extension' : extension not supported: a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhooooooooooooooooooooooooooooooohhhhhhhhhhhhhhhhh01234
@@ -29,7 +27,7 @@ ERROR: 0:62: 'preprocessor evaluation' : undefined macro in expression not allow
 ERROR: 0:67: '' : numeric literal too long 
 ERROR: 0:70: '' : name too long 
 ERROR: 0:70: 'preprocessor evaluation' : undefined macro in expression not allowed in es profile A000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
-ERROR: 28 compilation errors.  No code generated.
+ERROR: 26 compilation errors.  No code generated.
 
 
 Shader version: 300
@@ -73,7 +71,7 @@ ERROR: node is still EOpNull!
 0:23    move second child to first child (temp highp float)
 0:23      'E3' (global highp float)
 0:23      Constant:
-0:23        12.000000
+0:23        1.012346
 0:25  Function Definition: main( (global void)
 0:25    Function Parameters: 
 0:27    Sequence
@@ -104,7 +102,7 @@ ERROR: node is still EOpNull!
 0:36    move second child to first child (temp highp float)
 0:36      'superF' (global highp float)
 0:36      Constant:
-0:36        inf
+0:36        1.012346
 0:?   Linker Objects
 0:?     'BCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789' (in highp float)
 0:?     'ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789' (in highp float)
@@ -169,7 +167,7 @@ ERROR: node is still EOpNull!
 0:23    move second child to first child (temp highp float)
 0:23      'E3' (global highp float)
 0:23      Constant:
-0:23        12.000000
+0:23        1.012346
 0:25  Function Definition: main( (global void)
 0:25    Function Parameters: 
 0:27    Sequence
@@ -200,7 +198,7 @@ ERROR: node is still EOpNull!
 0:36    move second child to first child (temp highp float)
 0:36      'superF' (global highp float)
 0:36      Constant:
-0:36        inf
+0:36        1.012346
 0:?   Linker Objects
 0:?     'BCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789' (in highp float)
 0:?     'ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789' (in highp float)

glslang/Include/revision.h

@@ -2,5 +2,5 @@
 // For the version, it uses the latest git tag followed by the number of commits.
 // For the date, it uses the current date (when then script is run).
 
-#define GLSLANG_REVISION "Overload400-PrecQual.1760"
+#define GLSLANG_REVISION "Overload400-PrecQual.1764"
-#define GLSLANG_DATE "11-Jan-2017"
+#define GLSLANG_DATE "12-Jan-2017"

glslang/MachineIndependent/preprocessor/PpScanner.cpp

@@ -100,36 +100,26 @@ namespace glslang {
 int TPpContext::lFloatConst(int len, int ch, TPpToken* ppToken)
 {
     bool HasDecimalOrExponent = false;
-    int declen;
-    int str_len;
     int isDouble = 0;
 #ifdef AMD_EXTENSIONS
     int isFloat16 = 0;
     bool enableFloat16 = parseContext.version >= 450 && parseContext.extensionTurnedOn(E_GL_AMD_gpu_shader_half_float);
 #endif
 
-    declen = 0;
+    const auto saveName = [&](int ch) {
+        if (len <= MaxTokenLength)
+            ppToken->name[len++] = static_cast<char>(ch);
+    };
+
+    // Decimal:
 
-    str_len=len;
-    char* str = ppToken->name;
     if (ch == '.') {
         HasDecimalOrExponent = true;
-        str[len++] = (char)ch;
+        saveName(ch);
         ch = getChar();
         while (ch >= '0' && ch <= '9') {
-            if (len < MaxTokenLength) {
+            saveName(ch);
-                declen++;
-                if (len > 0 || ch != '0') {
-                    str[len] = (char)ch;
-                    len++;
-                    str_len++;
-                }
             ch = getChar();
-            } else {
-                parseContext.ppError(ppToken->loc, "float literal too long", "", "");
-                len = 1;
-                str_len = 1;
-            }
         }
     }
 
@@ -137,41 +127,24 @@ int TPpContext::lFloatConst(int len, int ch, TPpToken* ppToken)
 
     if (ch == 'e' || ch == 'E') {
         HasDecimalOrExponent = true;
-        if (len >= MaxTokenLength) {
+        saveName(ch);
-            parseContext.ppError(ppToken->loc, "float literal too long", "", "");
-            len = 1;
-            str_len = 1;
-        } else {
-            str[len++] = (char)ch;
         ch = getChar();
-            if (ch == '+') {
+        if (ch == '+' || ch == '-') {
-                str[len++] = (char)ch;
+            saveName(ch);
-                ch = getChar();
-            } else if (ch == '-') {
-                str[len++] = (char)ch;
             ch = getChar();
         }
         if (ch >= '0' && ch <= '9') {
             while (ch >= '0' && ch <= '9') {
-                    if (len < MaxTokenLength) {
+                saveName(ch);
-                        str[len++] = (char)ch;
                 ch = getChar();
-                    } else {
-                        parseContext.ppError(ppToken->loc, "float literal too long", "", "");
-                        len = 1;
-                        str_len = 1;
-                    }
             }
         } else {
             parseContext.ppError(ppToken->loc, "bad character in float exponent", "", "");
         }
     }
-    }
 
-    if (len == 0) {
+    // Suffix:
-        ppToken->dval = 0.0;
+
-        strcpy(str, "0.0");
-    } else {
     if (ch == 'l' || ch == 'L') {
         parseContext.doubleCheck(ppToken->loc, "double floating-point suffix");
         if (! HasDecimalOrExponent)
@@ -181,14 +154,9 @@ int TPpContext::lFloatConst(int len, int ch, TPpToken* ppToken)
             ungetChar();
             ungetChar();
         } else {
-                if (len < MaxTokenLength) {
+            saveName(ch);
-                    str[len++] = (char)ch;
+            saveName(ch2);
-                    str[len++] = (char)ch2;
             isDouble = 1;
-                } else {
-                    parseContext.ppError(ppToken->loc, "float literal too long", "", "");
-                    len = 1,str_len=1;
-                }
         }
 #ifdef AMD_EXTENSIONS
     } else if (enableFloat16 && (ch == 'h' || ch == 'H')) {
@@ -199,18 +167,11 @@ int TPpContext::lFloatConst(int len, int ch, TPpToken* ppToken)
         if (ch2 != 'f' && ch2 != 'F') {
             ungetChar();
             ungetChar();
-            }
+        } else {
-            else {
+            saveName(ch);
-                if (len < MaxTokenLength) {
+            saveName(ch2);
-                    str[len++] = (char)ch;
-                    str[len++] = (char)ch2;
             isFloat16 = 1;
         }
-                else {
-                    parseContext.ppError(ppToken->loc, "float literal too long", "", "");
-                    len = 1, str_len = 1;
-                }
-            }
 #endif
     } else if (ch == 'f' || ch == 'F') {
         parseContext.profileRequires(ppToken->loc,  EEsProfile, 300, nullptr, "floating-point suffix");
@@ -218,20 +179,22 @@ int TPpContext::lFloatConst(int len, int ch, TPpToken* ppToken)
             parseContext.profileRequires(ppToken->loc, ~EEsProfile, 120, nullptr, "floating-point suffix");
         if (! HasDecimalOrExponent)
             parseContext.ppError(ppToken->loc, "float literal needs a decimal point or exponent", "", "");
-            if (len < MaxTokenLength)
+        saveName(ch);
-                str[len++] = (char)ch;
-            else {
-                parseContext.ppError(ppToken->loc, "float literal too long", "", "");
-                len = 1,str_len=1;
-            }
     } else
         ungetChar();
 
-        str[len]='\0';
+    // Patch up the name, length, etc.
 
-        ppToken->dval = strtod(str, nullptr);
+    if (len > MaxTokenLength) {
+        len = MaxTokenLength;
+        parseContext.ppError(ppToken->loc, "float literal too long", "", "");
     }
+    ppToken->name[len] = '\0';
 
+    // Get the numerical value
+    ppToken->dval = strtod(ppToken->name, nullptr);
+
+    // Return the right token type
     if (isDouble)
         return PpAtomConstDouble;
 #ifdef AMD_EXTENSIONS