BeeEngine/glslang-zig
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Eliminate flex as the GLSL lexical analyzer, going from two nested lexical analyzers down to one, leaving just the preprocessor's lexical analysis. A new layer replaces it, to translate from the preprocessor's view of tokenization to glslang's view of tokenization.

Browse source 
Also:
 - change source locations from an int to TSourceLoc (shader number, line number) throughout
 - various improvements to the preprocessor


git-svn-id: https://cvs.khronos.org/svn/repos/ogl/trunk/ecosystem/public/sdk/tools/glslang@22277 e7fa87d3-cd2b-0410-9028-fcbf551c1848
This commit is contained in:
John Kessenich 2013-07-06 19:54:21 +00:00
parent 73ed17a87b
commit 5f1a0b7998
35 changed files with 2535 additions and 2515 deletions
Download patch file Download diff file Expand all files Collapse all files

17
glslang/MachineIndependent/preprocessor/atom.c
View file

@ -107,29 +107,20 @@ static const struct {
{ CPP_ADD_ASSIGN, "+=" },
{ CPP_DIV_ASSIGN, "/=" },
{ CPP_MUL_ASSIGN, "*=" },
{ CPP_RIGHT_BRACKET, ":>" },
{ CPP_EQ_OP, "==" },
{ CPP_XOR_OP, "^^" },
{ CPP_XOR_ASSIGN, "^=" },
{ CPP_FLOATCONSTANT, "<float-const>" },
{ CPP_GE_OP, ">=" },
{ CPP_RIGHT_OP, ">>" },
{ CPP_RIGHT_ASSIGN, ">>=" },
{ CPP_IDENTIFIER, "<ident>" },
{ CPP_INTCONSTANT, "<int-const>" },
{ CPP_RIGHT_ASSIGN, ">>="},
{ CPP_LE_OP, "<=" },
{ CPP_LEFT_OP, "<<" },
{ CPP_LEFT_ASSIGN, "<<=" },
{ CPP_LEFT_BRACKET, "<:" },
{ CPP_LEFT_BRACE, "<%" },
{ CPP_LEFT_ASSIGN, "<<="},
{ CPP_DEC_OP, "--" },
{ CPP_RIGHT_BRACE, "%>" },
{ CPP_NE_OP, "!=" },
{ CPP_OR_OP, "||" },
{ CPP_OR_ASSIGN, "|=" },
{ CPP_INC_OP, "++" },
{ CPP_STRCONSTANT, "<string-const>" },
{ CPP_TYPEIDENTIFIER, "<type-ident>" },
};
///////////////////////////////////////////////////////////////////////////////////////////////
@ -672,7 +663,7 @@ int InitAtomTable(AtomTable *atable, int htsize)
// Initialize lower part of atom table to "<undefined>" atom:
AddAtomFixed(atable, "<undefined>", 0);
for (ii = 0; ii < FIRST_USER_TOKEN_SY; ii++)
for (ii = 0; ii < CPP_FIRST_USER_TOKEN_SY; ii++)
atable->amap[ii] = atable->amap[0];
// Add single character tokens to the atom table:
@ -697,7 +688,7 @@ int InitAtomTable(AtomTable *atable, int htsize)
// Add error symbol if running in error mode:
if (cpp->options.ErrorMode)
AddAtomFixed(atable, "error", ERROR_SY);
AddAtomFixed(atable, "error", CPP_ERROR_SY);
AddAtom(atable, "<*** end fixed atoms ***>");

132
glslang/MachineIndependent/preprocessor/compile.h
View file

@ -1,132 +0,0 @@
//
//Copyright (C) 2002-2005 3Dlabs Inc. Ltd.
//All rights reserved.
//
//Redistribution and use in source and binary forms, with or without
//modification, are permitted provided that the following conditions
//are met:
//
// Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following
// disclaimer in the documentation and/or other materials provided
// with the distribution.
//
// Neither the name of 3Dlabs Inc. Ltd. nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
//THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
//"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
//LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
//FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
//COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
//INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
//BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
//LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
//CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
//LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
//ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
//POSSIBILITY OF SUCH DAMAGE.
//
/****************************************************************************\
Copyright (c) 2002, NVIDIA Corporation.
NVIDIA Corporation("NVIDIA") supplies this software to you in
consideration of your agreement to the following terms, and your use,
installation, modification or redistribution of this NVIDIA software
constitutes acceptance of these terms. If you do not agree with these
terms, please do not use, install, modify or redistribute this NVIDIA
software.
In consideration of your agreement to abide by the following terms, and
subject to these terms, NVIDIA grants you a personal, non-exclusive
license, under NVIDIA's copyrights in this original NVIDIA software (the
"NVIDIA Software"), to use, reproduce, modify and redistribute the
NVIDIA Software, with or without modifications, in source and/or binary
forms; provided that if you redistribute the NVIDIA Software, you must
retain the copyright notice of NVIDIA, this notice and the following
text and disclaimers in all such redistributions of the NVIDIA Software.
Neither the name, trademarks, service marks nor logos of NVIDIA
Corporation may be used to endorse or promote products derived from the
NVIDIA Software without specific prior written permission from NVIDIA.
Except as expressly stated in this notice, no other rights or licenses
express or implied, are granted by NVIDIA herein, including but not
limited to any patent rights that may be infringed by your derivative
works or by other works in which the NVIDIA Software may be
incorporated. No hardware is licensed hereunder.
THE NVIDIA SOFTWARE IS BEING PROVIDED ON AN "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED,
INCLUDING WITHOUT LIMITATION, WARRANTIES OR CONDITIONS OF TITLE,
NON-INFRINGEMENT, MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR
ITS USE AND OPERATION EITHER ALONE OR IN COMBINATION WITH OTHER
PRODUCTS.
IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT,
INCIDENTAL, EXEMPLARY, CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, LOST PROFITS; PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) OR ARISING IN ANY WAY
OUT OF THE USE, REPRODUCTION, MODIFICATION AND/OR DISTRIBUTION OF THE
NVIDIA SOFTWARE, HOWEVER CAUSED AND WHETHER UNDER THEORY OF CONTRACT,
TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, EVEN IF
NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\****************************************************************************/
//
// compile.h
//
#if !defined(__COMPILE_H)
#define __COMPILE_H 1
int InitCPPStruct(void);
typedef struct Options_Rec{
const char *profileString;
int ErrorMode;
int Quiet;
// Debug The Compiler options:
int DumpAtomTable;
} Options;
struct CPPStruct_Rec {
// Public members
SourceLoc *pLastSourceLoc; // Set at the start of each statement by the tree walkers
Options options; // Compile options and parameters
// Private members
SourceLoc lastSourceLoc;
// Scanner data:
SourceLoc *tokenLoc; // Source location of most recent token seen by the scanner
int mostRecentToken; // Most recent token seen by the scanner
InputSrc *currentInput;
int previous_token;
int notAVersionToken; // used to make sure that #version is the first token seen in the file, if present
void *pC; // storing the parseContext of the compile object in cpp.
// Private members:
SourceLoc ltokenLoc;
int ifdepth; //current #if-#else-#endif nesting in the cpp.c file (pre-processor)
int elsedepth[64]; //Keep a track of #if depth..Max allowed is 64.
int elsetracker; //#if-#else and #endif constructs...Counter.
const char *ErrMsg;
int CompileError; //Indicate compile error when #error, #else,#elif mismatch.
//
// Globals used to communicate between PaParseStrings() and yy_input()and
// also across the files.(gen_glslang.cpp and scanner.c)
//
int PaWhichStr; // which string we're parsing
int* PaStrLen; // array of lengths of the PaArgv strings
int PaArgc; // count of strings in the array
char** PaArgv; // our array of strings to parse
unsigned int tokensBeforeEOF : 1;
};
#endif // !defined(__COMPILE_H)

35
glslang/MachineIndependent/preprocessor/cpp.c
View file

@ -630,7 +630,8 @@ static int CPPerror(yystypepp * yylvalpp)
const char *message;
while (token != '\n') {
if (token == CPP_FLOATCONSTANT || token == CPP_INTCONSTANT){
if (token == CPP_INTCONSTANT || token == CPP_UINTCONSTANT ||
token == CPP_FLOATCONSTANT || token == CPP_DOUBLECONSTANT) {
StoreStr(yylvalpp->symbol_name);
}else if(token == CPP_IDENTIFIER || token == CPP_STRCONSTANT){
StoreStr(GetStringOfAtom(atable, yylvalpp->sc_ident));
@ -681,11 +682,9 @@ static int CPPpragma(yystypepp * yylvalpp)
strcpy(allTokens[tokenCount++], SrcStr);
break;
case CPP_INTCONSTANT:
SrcStr = yylvalpp->symbol_name;
allTokens[tokenCount] = (char*)malloc(strlen(SrcStr) + 1);
strcpy(allTokens[tokenCount++], SrcStr);
break;
case CPP_UINTCONSTANT:
case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT:
SrcStr = yylvalpp->symbol_name;
allTokens[tokenCount] = (char*)malloc(strlen(SrcStr) + 1);
strcpy(allTokens[tokenCount++], SrcStr);
@ -715,41 +714,35 @@ static int CPPpragma(yystypepp * yylvalpp)
return token;
} // CPPpragma
// This is just for error checking: the version and profile are decided before preprocessing starts
static int CPPversion(yystypepp * yylvalpp)
{
int token = cpp->currentInput->scan(cpp->currentInput, yylvalpp);
if (cpp->notAVersionToken == 1)
ShPpErrorToInfoLog("#version must occur before any other statement in the program");
if(token=='\n'){
if (token == '\n'){
DecLineNumber();
ShPpErrorToInfoLog("#version");
IncLineNumber();
return token;
}
if (token != CPP_INTCONSTANT)
ShPpErrorToInfoLog("#version");
yylvalpp->sc_int=atoi(yylvalpp->symbol_name);
SetVersion(yylvalpp->sc_int);
yylvalpp->sc_int = atoi(yylvalpp->symbol_name);
token = cpp->currentInput->scan(cpp->currentInput, yylvalpp);
if (token == '\n') {
SetProfile(ENoProfile);
if (token == '\n')
return token;
}
else {
if (yylvalpp->sc_ident == coreAtom)
SetProfile(ECoreProfile);
else if (yylvalpp->sc_ident == compatibilityAtom)
SetProfile(ECompatibilityProfile);
else if (yylvalpp->sc_ident == esAtom)
SetProfile(EEsProfile);
else
if (yylvalpp->sc_ident != coreAtom &&
yylvalpp->sc_ident != compatibilityAtom &&
yylvalpp->sc_ident != esAtom)
ShPpErrorToInfoLog("#version profile name");
token = cpp->currentInput->scan(cpp->currentInput, yylvalpp);
@ -1025,7 +1018,7 @@ int MacroExpand(int atom, yystypepp* yylvalpp, int expandUndef)
}
if (atom == __VERSION__Atom) {
yylvalpp->sc_int = GetVersion(cpp->pC);
yylvalpp->sc_int = GetShaderVersion(cpp->pC);
sprintf(yylvalpp->symbol_name, "%d", yylvalpp->sc_int);
UngetToken(CPP_INTCONSTANT, yylvalpp);

3
glslang/MachineIndependent/preprocessor/cpp.h
View file

@ -116,8 +116,7 @@ int GetLineNumber(void); // Get the current String Number.
int GetStringNumber(void); // Get the current String Number.
const char* GetStrfromTStr(void); // Convert TString to String.
void SetVersion(int);
void SetProfile(EProfile);
int GetVersion(void*);
int GetShaderVersion(void*);
void updateExtensionBehavior(const char* extName, const char* behavior);
int FreeCPP(void);

2
glslang/MachineIndependent/preprocessor/cppstruct.c
View file

@ -86,10 +86,8 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
CPPStruct *cpp = NULL;
static int refCount = 0;
int InitPreprocessor(void);
int ResetPreprocessor(void);
int FreeCPPStruct(void);
int FinalizePreprocessor(void);
/*
* InitCPPStruct() - Initilaize the CPP structure.

25
glslang/MachineIndependent/preprocessor/parser.h
View file

@ -75,20 +75,9 @@ TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, EVEN IF
NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\****************************************************************************/
#ifndef BISON_PARSER_H
# define BISON_PARSER_H
#ifndef PARSER_H
# define PARSER_H
#ifndef yystypepp
typedef struct {
int sc_int;
float sc_fval;
double sc_dval;
int sc_ident;
char symbol_name[MAX_SYMBOL_NAME_LEN+1];
} yystypepp;
# define YYSTYPE_IS_TRIVIAL 1
#endif
# define CPP_AND_OP 257
# define CPP_SUB_ASSIGN 259
# define CPP_MOD_ASSIGN 260
@ -97,7 +86,7 @@ typedef struct {
# define CPP_MUL_ASSIGN 263
# define CPP_EQ_OP 264
# define CPP_XOR_OP 265
# define ERROR_SY 266
# define CPP_ERROR_SY 266
# define CPP_FLOATCONSTANT 267
# define CPP_GE_OP 268
# define CPP_RIGHT_OP 269
@ -111,9 +100,6 @@ typedef struct {
# define CPP_INC_OP 277
# define CPP_STRCONSTANT 278
# define CPP_TYPEIDENTIFIER 279
# define FIRST_USER_TOKEN_SY 289
# define CPP_RIGHT_ASSIGN 280
# define CPP_LEFT_ASSIGN 281
# define CPP_AND_ASSIGN 282
@ -123,5 +109,8 @@ typedef struct {
# define CPP_RIGHT_BRACKET 286
# define CPP_LEFT_BRACE 287
# define CPP_RIGHT_BRACE 288
# define CPP_UINTCONSTANT 289
# define CPP_DOUBLECONSTANT 290
# define CPP_FIRST_USER_TOKEN_SY 291
#endif /* not BISON_PARSER_H */
#endif /* not PARSER_H */

84
glslang/MachineIndependent/preprocessor/preprocess.h
View file

@ -75,10 +75,84 @@ TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, EVEN IF
NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\****************************************************************************/
#include "slglobals.h"
#ifndef PREPROCESS_H
#define PREPROCESS_H
typedef struct SourceLoc_Rec {
int file;
int line;
} SourceLoc;
typedef struct Options_Rec {
const char *profileString;
int ErrorMode;
int Quiet;
// Debug The Compiler options:
int DumpAtomTable;
} Options;
#define MAX_TOKEN_LENGTH 1024
typedef struct {
int ppToken;
int sc_int;
double sc_dval;
int sc_ident;
char symbol_name[MAX_TOKEN_LENGTH+1];
} yystypepp;
typedef struct InputSrc {
struct InputSrc *prev;
int (*scan)(struct InputSrc *, yystypepp *);
int (*getch)(struct InputSrc *, yystypepp *);
void (*ungetch)(struct InputSrc *, int, yystypepp *);
int name; /* atom */
int line;
} InputSrc;
typedef struct CPPStruct {
// Public members
SourceLoc *pLastSourceLoc; // Set at the start of each statement by the tree walkers
Options options; // Compile options and parameters
// Private members
SourceLoc lastSourceLoc;
// Scanner data:
SourceLoc *tokenLoc; // Source location of most recent token seen by the scanner
int mostRecentToken; // Most recent token seen by the scanner
InputSrc *currentInput;
int previous_token;
int notAVersionToken; // used to make sure that #version is the first token seen in the file, if present
void *pC; // storing the parseContext of the compile object in cpp.
// Private members:
SourceLoc ltokenLoc;
int ifdepth; //current #if-#else-#endif nesting in the cpp.c file (pre-processor)
int elsedepth[64]; //Keep a track of #if depth..Max allowed is 64.
int elsetracker; //#if-#else and #endif constructs...Counter.
const char *ErrMsg;
int CompileError; //Indicate compile error when #error, #else,#elif mismatch.
//
// Globals used to communicate between parseStrings() and yy_input()and
// also across the files.(gen_glslang.cpp and scanner.c)
//
int PaWhichStr; // which string we're parsing
int* PaStrLen; // array of lengths of the PaArgv strings
int PaArgc; // count of strings in the array
char** PaArgv; // our array of strings to parse
unsigned int tokensBeforeEOF : 1;
} CPPStruct;
extern CPPStruct *cpp;
int InitCPPStruct(void);
int InitScanner(CPPStruct *cpp);
int InitAtomTable(AtomTable *atable, int htsize);
int InitPreprocessor(void);
int FinalizePreprocessor(void);
int ScanFromString(char *s);
char* GetStringOfAtom(AtomTable *atable, int atom);
const char* PpTokenize(yystypepp*);
#endif

183
glslang/MachineIndependent/preprocessor/scanner.c
View file

@ -222,7 +222,7 @@ int ScanFromString(char *s)
cpp->currentInput = &in->base;
return 1;
} // ScanFromString;
}
///////////////////////////////////////////////////////////////////////////////////////////////
@ -239,6 +239,7 @@ static int lFloatConst(char *str, int len, int ch, yystypepp * yylvalpp)
{
int HasDecimal, declen, exp, ExpSign;
int str_len;
int isDouble = 0;
HasDecimal = 0;
declen = 0;
@ -250,7 +251,7 @@ static int lFloatConst(char *str, int len, int ch, yystypepp * yylvalpp)
HasDecimal = 1;
ch = cpp->currentInput->getch(cpp->currentInput, yylvalpp);
while (ch >= '0' && ch <= '9') {
if (len < MAX_SYMBOL_NAME_LEN) {
if (len < MAX_TOKEN_LENGTH) {
declen++;
if (len > 0 || ch != '0') {
str[len] = ch;
@ -267,7 +268,7 @@ static int lFloatConst(char *str, int len, int ch, yystypepp * yylvalpp)
// Exponent:
if (ch == 'e' || ch == 'E') {
if (len >= MAX_SYMBOL_NAME_LEN) {
if (len >= MAX_TOKEN_LENGTH) {
ShPpErrorToInfoLog("floating-point literal too long");
len = 1,str_len=1;
} else {
@ -284,7 +285,7 @@ static int lFloatConst(char *str, int len, int ch, yystypepp * yylvalpp)
}
if (ch >= '0' && ch <= '9') {
while (ch >= '0' && ch <= '9') {
if (len < MAX_SYMBOL_NAME_LEN) {
if (len < MAX_TOKEN_LENGTH) {
exp = exp*10 + ch - '0';
str[len++]=ch;
ch = cpp->currentInput->getch(cpp->currentInput, yylvalpp);
@ -301,7 +302,6 @@ static int lFloatConst(char *str, int len, int ch, yystypepp * yylvalpp)
}
if (len == 0) {
yylvalpp->sc_fval = 0.0f;
yylvalpp->sc_dval = 0.0;
strcpy(str, "0.0");
} else {
@ -311,16 +311,17 @@ static int lFloatConst(char *str, int len, int ch, yystypepp * yylvalpp)
cpp->currentInput->ungetch(cpp->currentInput, ch2, yylvalpp);
cpp->currentInput->ungetch(cpp->currentInput, ch, yylvalpp);
} else {
if (len < MAX_SYMBOL_NAME_LEN) {
if (len < MAX_TOKEN_LENGTH) {
str[len++] = ch;
str[len++] = ch2;
isDouble = 1;
} else {
ShPpErrorToInfoLog("floating-point literal too long");
len = 1,str_len=1;
}
}
} else if (ch == 'f' || ch == 'F') {
if (len < MAX_SYMBOL_NAME_LEN)
if (len < MAX_TOKEN_LENGTH)
str[len++] = ch;
else {
ShPpErrorToInfoLog("floating-point literal too long");
@ -332,12 +333,14 @@ static int lFloatConst(char *str, int len, int ch, yystypepp * yylvalpp)
str[len]='\0';
yylvalpp->sc_dval = strtod(str, 0);
yylvalpp->sc_fval = (float)yylvalpp->sc_dval;
}
// Suffix:
strcpy(yylvalpp->symbol_name, str);
return CPP_FLOATCONSTANT;
if (isDouble)
return CPP_DOUBLECONSTANT;
else
return CPP_FLOATCONSTANT;
} // lFloatConst
///////////////////////////////////////////////////////////////////////////////////////////////
@ -346,8 +349,7 @@ static int lFloatConst(char *str, int len, int ch, yystypepp * yylvalpp)
static int byte_scan(InputSrc *in, yystypepp * yylvalpp)
{
char symbol_name[MAX_SYMBOL_NAME_LEN + 1];
char string_val[MAX_STRING_LEN + 1];
char tokenText[MAX_TOKEN_LENGTH + 1];
int AlreadyComplained = 0;
int len, ch, ii;
unsigned ival = 0;
@ -368,7 +370,7 @@ static int byte_scan(InputSrc *in, yystypepp * yylvalpp)
default:
return ch; // Single character token
case EOF:
return -1;
return EOF;
case 'A': case 'B': case 'C': case 'D': case 'E':
case 'F': case 'G': case 'H': case 'I': case 'J':
case 'K': case 'L': case 'M': case 'N': case 'O':
@ -393,30 +395,32 @@ static int byte_scan(InputSrc *in, yystypepp * yylvalpp)
ch = nextch;
} else
ShPpErrorToInfoLog("can only escape newlines");
} else if (len < MAX_SYMBOL_NAME_LEN) {
symbol_name[len] = ch;
len++;
} else if (len < MAX_TOKEN_LENGTH) {
tokenText[len++] = ch;
ch = cpp->currentInput->getch(cpp->currentInput, yylvalpp);
} else {
ShPpErrorToInfoLog("name too long");
break;
if (! AlreadyComplained) {
ShPpErrorToInfoLog("name too long");
AlreadyComplained = 1;
}
ch = cpp->currentInput->getch(cpp->currentInput, yylvalpp);
}
} while ((ch >= 'a' && ch <= 'z') ||
(ch >= 'A' && ch <= 'Z') ||
(ch >= '0' && ch <= '9') ||
ch == '_' ||
ch == '\\');
if (len > MAX_SYMBOL_NAME_LEN)
len = MAX_SYMBOL_NAME_LEN;
symbol_name[len] = '\0';
tokenText[len] = '\0';
cpp->currentInput->ungetch(cpp->currentInput, ch, yylvalpp);
yylvalpp->sc_ident = LookUpAddString(atable, symbol_name);
yylvalpp->sc_ident = LookUpAddString(atable, tokenText);
return CPP_IDENTIFIER;
break;
case '0':
yylvalpp->symbol_name[len++] = ch;
ch = cpp->currentInput->getch(cpp->currentInput, yylvalpp);
if (ch == 'x' || ch == 'X') {
int uint = 0;
yylvalpp->symbol_name[len++] = ch;
ch = cpp->currentInput->getch(cpp->currentInput, yylvalpp);
if ((ch >= '0' && ch <= '9') ||
@ -425,10 +429,8 @@ static int byte_scan(InputSrc *in, yystypepp * yylvalpp)
{
ival = 0;
do {
if (len >= MAX_SYMBOL_NAME_LEN)
break;
yylvalpp->symbol_name[len++] = ch;
if (ival <= 0x0fffffff) {
yylvalpp->symbol_name[len++] = ch;
if (ch >= '0' && ch <= '9') {
ii = ch - '0';
} else if (ch >= 'A' && ch <= 'F') {
@ -440,9 +442,10 @@ static int byte_scan(InputSrc *in, yystypepp * yylvalpp)
ival = (ival << 4) | ii;
} else {
if (! AlreadyComplained) {
ShPpErrorToInfoLog("hexidecimal literal too long");
ShPpErrorToInfoLog("hexidecimal literal too big");
AlreadyComplained = 1;
}
ival = 0xffffffff;
}
ch = cpp->currentInput->getch(cpp->currentInput, yylvalpp);
} while ((ch >= '0' && ch <= '9') ||
@ -451,38 +454,51 @@ static int byte_scan(InputSrc *in, yystypepp * yylvalpp)
} else {
ShPpErrorToInfoLog("bad digit in hexidecimal literal");
}
if (ch == 'u' || ch == 'U')
yylvalpp->symbol_name[len++] = ch;
else
if (ch == 'u' || ch == 'U') {
if (len < MAX_TOKEN_LENGTH)
yylvalpp->symbol_name[len++] = ch;
uint = 1;
} else
cpp->currentInput->ungetch(cpp->currentInput, ch, yylvalpp);
yylvalpp->symbol_name[len] = '\0';
yylvalpp->sc_int = (int)ival;
return CPP_INTCONSTANT;
if (uint)
return CPP_UINTCONSTANT;
else
return CPP_INTCONSTANT;
} else if (ch >= '0' && ch <= '7') { // octal integer constants
int uint = 0;
ival = 0;
do {
if (len >= MAX_SYMBOL_NAME_LEN)
break;
yylvalpp->symbol_name[len++] = ch;
if (ival <= 0x1fffffff) {
yylvalpp->symbol_name[len++] = ch;
ii = ch - '0';
ival = (ival << 3) | ii;
} else {
if (!AlreadyComplained) {
ShPpErrorToInfoLog("octal literal too long");
ShPpErrorToInfoLog("octal literal too big");
AlreadyComplained = 1;
}
ival = 0xffffffff;
}
ch = cpp->currentInput->getch(cpp->currentInput, yylvalpp);
} while (ch >= '0' && ch <= '7');
if (ch == '.' || ch == 'e' || ch == 'f' || ch == 'h' || ch == 'x'|| ch == 'E' || ch == 'F' || ch == 'l' || ch == 'L')
return lFloatConst(yylvalpp->symbol_name, len, ch, yylvalpp);
else if (ch == 'u' || ch == 'U') {
if (len < MAX_TOKEN_LENGTH)
yylvalpp->symbol_name[len++] = ch;
uint = 1;
} else
cpp->currentInput->ungetch(cpp->currentInput, ch, yylvalpp);
yylvalpp->symbol_name[len] = '\0';
cpp->currentInput->ungetch(cpp->currentInput, ch, yylvalpp);
yylvalpp->sc_int = (int)ival;
return CPP_INTCONSTANT;
if (uint)
return CPP_UINTCONSTANT;
else
return CPP_INTCONSTANT;
} else {
cpp->currentInput->ungetch(cpp->currentInput, ch, yylvalpp);
ch = '0';
@ -491,25 +507,30 @@ static int byte_scan(InputSrc *in, yystypepp * yylvalpp)
case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
do {
if (len < MAX_SYMBOL_NAME_LEN) {
if (len < MAX_TOKEN_LENGTH) {
if (len > 0 || ch != '0') {
yylvalpp->symbol_name[len] = ch;
len++;
}
ch = cpp->currentInput->getch(cpp->currentInput, yylvalpp);
} else {
ShPpErrorToInfoLog("token too long");
break;
if (! AlreadyComplained) {
ShPpErrorToInfoLog("integer literal too long");
AlreadyComplained = 1;
}
}
ch = cpp->currentInput->getch(cpp->currentInput, yylvalpp);
} while (ch >= '0' && ch <= '9');
if (ch == '.' || ch == 'e' || ch == 'f' || ch == 'h' || ch == 'x'|| ch == 'E' || ch == 'F' || ch == 'l' || ch == 'L') {
return lFloatConst(yylvalpp->symbol_name, len, ch, yylvalpp);
} else {
// Finish handling signed and unsigned integers
int numericLen = len;
if (ch == 'u' || ch == 'U')
yylvalpp->symbol_name[len++] = ch;
else
int uint = 0;
if (ch == 'u' || ch == 'U') {
if (len < MAX_TOKEN_LENGTH)
yylvalpp->symbol_name[len++] = ch;
uint = 1;
} else
cpp->currentInput->ungetch(cpp->currentInput, ch, yylvalpp);
yylvalpp->symbol_name[len] = '\0';
@ -517,16 +538,18 @@ static int byte_scan(InputSrc *in, yystypepp * yylvalpp)
for (ii = 0; ii < numericLen; ii++) {
ch = yylvalpp->symbol_name[ii] - '0';
if ((ival > 429496729) || (ival == 429496729 && ch >= 6)) {
if (! AlreadyComplained) {
ShPpErrorToInfoLog("integral literal too long");
AlreadyComplained = 1;
}
}
ival = ival * 10 + ch;
ShPpErrorToInfoLog("integral literal too big");
ival = -1;
break;
} else
ival = ival * 10 + ch;
}
yylvalpp->sc_int = (int)ival;
return CPP_INTCONSTANT;
if (uint)
return CPP_UINTCONSTANT;
else
return CPP_INTCONSTANT;
}
break;
case '-':
@ -737,39 +760,39 @@ static int byte_scan(InputSrc *in, yystypepp * yylvalpp)
break;
}
}
if (len < MAX_STRING_LEN) {
string_val[len] = ch;
if (len < MAX_TOKEN_LENGTH) {
tokenText[len] = ch;
len++;
ch = cpp->currentInput->getch(cpp->currentInput, yylvalpp);
} else
break;
};
string_val[len] = '\0';
tokenText[len] = '\0';
if (ch == '"') {
yylvalpp->sc_ident = LookUpAddString(atable, string_val);
yylvalpp->sc_ident = LookUpAddString(atable, tokenText);
return CPP_STRCONSTANT;
} else {
ShPpErrorToInfoLog("end of line in string");
return ERROR_SY;
return CPP_ERROR_SY;
}
}
}
} // byte_scan
int yylex_CPP(char* buf, int maxSize)
const char* PpTokenize(yystypepp* yylvalpp)
{
yystypepp yylvalpp;
int token = '\n';
for(;;) {
char* tokenString = 0;
token = cpp->currentInput->scan(cpp->currentInput, &yylvalpp);
if(check_EOF(token))
token = cpp->currentInput->scan(cpp->currentInput, yylvalpp);
yylvalpp->ppToken = token;
if (check_EOF(token))
return 0;
if (token == '#') {
if (cpp->previous_token == '\n'|| cpp->previous_token == 0) {
token = readCPPline(&yylvalpp);
token = readCPPline(yylvalpp);
if(check_EOF(token))
return 0;
continue;
@ -779,42 +802,34 @@ int yylex_CPP(char* buf, int maxSize)
}
}
cpp->previous_token = token;
// expand macros
if (token == CPP_IDENTIFIER && MacroExpand(yylvalpp.sc_ident, &yylvalpp, 0) == 1) {
cpp->notAVersionToken = 1;
continue;
}
if (token == '\n')
continue;
if (token == CPP_IDENTIFIER) {
cpp->notAVersionToken = 1;
tokenString = GetStringOfAtom(atable,yylvalpp.sc_ident);
} else if (token == CPP_FLOATCONSTANT||token == CPP_INTCONSTANT){
cpp->notAVersionToken = 1;
tokenString = yylvalpp.symbol_name;
} else {
cpp->notAVersionToken = 1;
tokenString = GetStringOfAtom(atable,token);
}
cpp->notAVersionToken = 1;
// expand macros
if (token == CPP_IDENTIFIER && MacroExpand(yylvalpp->sc_ident, yylvalpp, 0) == 1)
continue;
if (token == CPP_IDENTIFIER)
tokenString = GetStringOfAtom(atable, yylvalpp->sc_ident);
else if (token == CPP_INTCONSTANT || token == CPP_UINTCONSTANT ||
token == CPP_FLOATCONSTANT || token == CPP_DOUBLECONSTANT)
tokenString = yylvalpp->symbol_name;
else
tokenString = GetStringOfAtom(atable, token);
if (tokenString) {
if ((signed)strlen(tokenString) >= maxSize) {
if (tokenString[0] != 0)
cpp->tokensBeforeEOF = 1;
return maxSize;
} else if (strlen(tokenString) > 0) {
strcpy(buf, tokenString);
cpp->tokensBeforeEOF = 1;
return (int)strlen(tokenString);
}
return 0;
return tokenString;
}
}
return 0;
} // yylex
} // PpTokenize
//Checks if the token just read is EOF or not.
int check_EOF(int token)

19
glslang/MachineIndependent/preprocessor/scanner.h
View file

@ -81,9 +81,7 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#if !defined(__SCANNER_H)
#define __SCANNER_H 1
#define MAX_SYMBOL_NAME_LEN 1025
#define MAX_STRING_LEN 1025
#include "preprocess.h"
#include "parser.h"
// Not really atom table stuff but needed first...
@ -91,23 +89,8 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
extern "C" {
#endif
typedef struct SourceLoc_Rec {
unsigned short file, line;
} SourceLoc;
int yyparse (void);
int yylex_CPP(char* buf, int maxSize);
typedef struct InputSrc {
struct InputSrc *prev;
int (*scan)(struct InputSrc *, yystypepp *);
int (*getch)(struct InputSrc *, yystypepp *);
void (*ungetch)(struct InputSrc *, int, yystypepp *);
int name; /* atom */
int line;
} InputSrc;
int InitScanner(CPPStruct *cpp); // Intialise the cpp scanner.
int ScanFromString(char *); // Start scanning the input from the string mentioned.
int check_EOF(int); // check if we hit a EOF abruptly

3
glslang/MachineIndependent/preprocessor/slglobals.h
View file

@ -81,7 +81,7 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#if !defined(__SLGLOBALS_H)
#define __SLGLOBALS_H 1
typedef struct CPPStruct_Rec CPPStruct;
#include "preprocess.h"
// TODO: threading: Multi-threading note: The existence of this global makes
// this preprocessing single-threaded only.
@ -101,7 +101,6 @@ extern CPPStruct *cpp;
#include "cpp.h"
#include "tokens.h"
#include "symbols.h"
#include "compile.h"
#if !defined(NO_PARSER)
#include "parser.h"
#endif

59
glslang/MachineIndependent/preprocessor/tokens.c
View file

@ -257,8 +257,10 @@ void RecordToken(TokenStream *pTok, int token, yystypepp * yylvalpp)
lAddByte(pTok, (unsigned char) *s++);
lAddByte(pTok, 0);
break;
case CPP_FLOATCONSTANT:
case CPP_INTCONSTANT:
case CPP_UINTCONSTANT:
case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT:
str = yylvalpp->symbol_name;
while (*str){
lAddByte(pTok, (unsigned char) *str);
@ -293,8 +295,9 @@ void RewindTokenStream(TokenStream *pTok)
int ReadToken(TokenStream *pTok, yystypepp * yylvalpp)
{
char symbol_name[MAX_SYMBOL_NAME_LEN + 1];
char string_val[MAX_STRING_LEN + 1];
//TODO: PP: why is this different than byte_scan
char tokenText[MAX_TOKEN_LENGTH + 1];
int ltoken, len;
char ch;
@ -312,8 +315,8 @@ int ReadToken(TokenStream *pTok, yystypepp * yylvalpp)
(ch >= '0' && ch <= '9') ||
ch == '_')
{
if (len < MAX_SYMBOL_NAME_LEN) {
symbol_name[len] = ch;
if (len < MAX_TOKEN_LENGTH) {
tokenText[len] = ch;
len++;
ch = lReadByte(pTok);
} else {
@ -321,30 +324,31 @@ int ReadToken(TokenStream *pTok, yystypepp * yylvalpp)
break;
}
}
symbol_name[len] = '\0';
tokenText[len] = '\0';
assert(ch == '\0');
yylvalpp->sc_ident = LookUpAddString(atable, symbol_name);
yylvalpp->sc_ident = LookUpAddString(atable, tokenText);
return CPP_IDENTIFIER;
break;
case CPP_STRCONSTANT:
len = 0;
while ((ch = lReadByte(pTok)) != 0) {
if (len < MAX_STRING_LEN)
string_val[len++] = ch;
if (len < MAX_TOKEN_LENGTH)
tokenText[len++] = ch;
else
break;
}
string_val[len] = 0;
yylvalpp->sc_ident = LookUpAddString(atable, string_val);
tokenText[len] = 0;
yylvalpp->sc_ident = LookUpAddString(atable, tokenText);
break;
case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT:
len = 0;
ch = lReadByte(pTok);
while ((ch >= '0' && ch <= '9')||(ch=='e'||ch=='E'||ch=='.')||(ch=='+'||ch=='-'))
while ((ch >= '0' && ch <= '9') || ch=='e' || ch=='E' || ch=='.' || ch=='+' || ch=='-' || ch=='l' || ch=='L' || ch=='f'|| ch=='F')
{
if (len < MAX_SYMBOL_NAME_LEN) {
symbol_name[len] = ch;
if (len < MAX_TOKEN_LENGTH) {
tokenText[len] = ch;
len++;
ch = lReadByte(pTok);
} else {
@ -352,18 +356,19 @@ int ReadToken(TokenStream *pTok, yystypepp * yylvalpp)
break;
}
}
symbol_name[len] = '\0';
tokenText[len] = '\0';
assert(ch == '\0');
strcpy(yylvalpp->symbol_name,symbol_name);
yylvalpp->sc_fval=(float)atof(yylvalpp->symbol_name);
strcpy(yylvalpp->symbol_name, tokenText);
yylvalpp->sc_dval = atof(yylvalpp->symbol_name);
break;
case CPP_INTCONSTANT:
case CPP_UINTCONSTANT:
len = 0;
ch = lReadByte(pTok);
while ((ch >= '0' && ch <= '9'))
while ((ch >= '0' && ch <= '9') || ch == 'u' || ch == 'U')
{
if (len < MAX_SYMBOL_NAME_LEN) {
symbol_name[len] = ch;
if (len < MAX_TOKEN_LENGTH) {
tokenText[len] = ch;
len++;
ch = lReadByte(pTok);
} else {
@ -371,10 +376,10 @@ int ReadToken(TokenStream *pTok, yystypepp * yylvalpp)
break;
}
}
symbol_name[len] = '\0';
tokenText[len] = '\0';
assert(ch == '\0');
strcpy(yylvalpp->symbol_name,symbol_name);
yylvalpp->sc_int=atoi(yylvalpp->symbol_name);
strcpy(yylvalpp->symbol_name,tokenText);
yylvalpp->sc_int = atoi(yylvalpp->symbol_name);
break;
case '(':
yylvalpp->sc_int = lReadByte(pTok);
@ -456,7 +461,7 @@ void UngetToken(int token, yystypepp * yylvalpp) {
void DumpTokenStream(FILE *fp, TokenStream *s, yystypepp * yylvalpp) {
int token;
const int maxSize = MAX_SYMBOL_NAME_LEN + 5;
const int maxSize = MAX_TOKEN_LENGTH + 5;
char str[100];
if (fp == 0) fp = stdout;
@ -471,10 +476,12 @@ void DumpTokenStream(FILE *fp, TokenStream *s, yystypepp * yylvalpp) {
snprintf(str, maxSize, "\"%s\"", GetAtomString(atable, yylvalpp->sc_ident));
break;
case CPP_FLOATCONSTANT:
//printf("%g9.6 ", yylvalpp->sc_fval);
case CPP_DOUBLECONSTANT:
printf("%g9.6 ", yylvalpp->sc_dval);
break;
case CPP_INTCONSTANT:
//printf("%d ", yylvalpp->sc_int);
case CPP_UINTCONSTANT:
printf("%d ", yylvalpp->sc_int);
break;
default:
if (token >= 127)