BeeEngine/glslang-zig
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Preprocessor: 1) Rationalize the "fixed atom" scheme, 2) remove redundant lookups when the text is already available.

Browse source 
This simplification is a prelude to eliminating what I appear unnecessary
symbol inserts into tables when tokenizing in the preprecessor, which
show up as taking notable time. (Performance issue.)  It also simply makes
the preprocessor easier to understand, which it is badly in need of.
This commit is contained in:
John Kessenich 2015-07-19 15:49:03 -06:00
parent 2f273369e4
commit 6ab3d582d6
10 changed files with 362 additions and 412 deletions
Download patch file Download diff file Expand all files Collapse all files

270
glslang/MachineIndependent/preprocessor/Pp.cpp
View file

@ -94,32 +94,6 @@ namespace glslang {
int TPpContext::InitCPP()
{
// Add various atoms needed by the CPP line scanner:
bindAtom = LookUpAddString("bind");
constAtom = LookUpAddString("const");
defaultAtom = LookUpAddString("default");
defineAtom = LookUpAddString("define");
definedAtom = LookUpAddString("defined");
elifAtom = LookUpAddString("elif");
elseAtom = LookUpAddString("else");
endifAtom = LookUpAddString("endif");
ifAtom = LookUpAddString("if");
ifdefAtom = LookUpAddString("ifdef");
ifndefAtom = LookUpAddString("ifndef");
includeAtom = LookUpAddString("include");
lineAtom = LookUpAddString("line");
pragmaAtom = LookUpAddString("pragma");
texunitAtom = LookUpAddString("texunit");
undefAtom = LookUpAddString("undef");
errorAtom = LookUpAddString("error");
__LINE__Atom = LookUpAddString("__LINE__");
__FILE__Atom = LookUpAddString("__FILE__");
__VERSION__Atom = LookUpAddString("__VERSION__");
versionAtom = LookUpAddString("version");
coreAtom = LookUpAddString("core");
compatibilityAtom = LookUpAddString("compatibility");
esAtom = LookUpAddString("es");
extensionAtom = LookUpAddString("extension");
pool = mem_CreatePool(0, 0);
return 1;
@ -133,17 +107,18 @@ int TPpContext::CPPdefine(TPpToken* ppToken)
// get macro name
int token = scanToken(ppToken);
if (token != CPP_IDENTIFIER) {
if (token != PpAtomIdentifier) {
parseContext.ppError(ppToken->loc, "must be followed by macro name", "#define", "");
return token;
}
int atom = ppToken->atom;
const char* definedName = GetAtomString(atom);
if (ppToken->loc.string >= 0) {
// We are in user code; check for reserved name use:
parseContext.reservedPpErrorCheck(ppToken->loc, definedName, "#define");
parseContext.reservedPpErrorCheck(ppToken->loc, ppToken->name, "#define");
}
// save the original atom
const int defAtom = ppToken->atom;
// gather parameters to the macro, between (...)
token = scanToken(ppToken);
if (token == '(' && ! ppToken->space) {
@ -153,7 +128,7 @@ int TPpContext::CPPdefine(TPpToken* ppToken)
token = scanToken(ppToken);
if (argc == 0 && token == ')')
break;
if (token != CPP_IDENTIFIER) {
if (token != PpAtomIdentifier) {
parseContext.ppError(ppToken->loc, "bad argument", "#define", "");
return token;
@ -197,18 +172,18 @@ int TPpContext::CPPdefine(TPpToken* ppToken)
}
// check for duplicate definition
symb = LookUpSymbol(atom);
symb = LookUpSymbol(defAtom);
if (symb) {
if (! symb->mac.undef) {
// Already defined -- need to make sure they are identical:
// "Two replacement lists are identical if and only if the preprocessing tokens in both have the same number,
// ordering, spelling, and white-space separation, where all white-space separations are considered identical."
if (symb->mac.argc != mac.argc)
parseContext.ppError(defineLoc, "Macro redefined; different number of arguments:", "#define", GetAtomString(atom));
parseContext.ppError(defineLoc, "Macro redefined; different number of arguments:", "#define", GetAtomString(defAtom));
else {
for (int argc = 0; argc < mac.argc; argc++) {
if (symb->mac.args[argc] != mac.args[argc])
parseContext.ppError(defineLoc, "Macro redefined; different argument names:", "#define", GetAtomString(atom));
parseContext.ppError(defineLoc, "Macro redefined; different argument names:", "#define", GetAtomString(defAtom));
}
RewindTokenStream(symb->mac.body);
RewindTokenStream(mac.body);
@ -220,14 +195,14 @@ int TPpContext::CPPdefine(TPpToken* ppToken)
oldToken = ReadToken(symb->mac.body, &oldPpToken);
newToken = ReadToken(mac.body, &newPpToken);
if (oldToken != newToken || oldPpToken != newPpToken) {
parseContext.ppError(defineLoc, "Macro redefined; different substitutions:", "#define", GetAtomString(atom));
parseContext.ppError(defineLoc, "Macro redefined; different substitutions:", "#define", GetAtomString(defAtom));
break;
}
} while (newToken > 0);
}
}
} else
symb = AddSymbol(atom);
symb = AddSymbol(defAtom);
delete symb->mac.body;
symb->mac = mac;
@ -240,14 +215,13 @@ int TPpContext::CPPundef(TPpToken* ppToken)
{
int token = scanToken(ppToken);
Symbol *symb;
if (token != CPP_IDENTIFIER) {
if (token != PpAtomIdentifier) {
parseContext.ppError(ppToken->loc, "must be followed by macro name", "#undef", "");
return token;
}
const char* name = GetAtomString(ppToken->atom); // TODO preprocessor simplification: the token text should have been built into the ppToken during scanToken()
parseContext.reservedPpErrorCheck(ppToken->loc, name, "#undef");
parseContext.reservedPpErrorCheck(ppToken->loc, ppToken->name, "#undef");
symb = LookUpSymbol(ppToken->atom);
if (symb) {
@ -283,15 +257,15 @@ int TPpContext::CPPelse(int matchelse, TPpToken* ppToken)
continue;
}
if ((token = scanToken(ppToken)) != CPP_IDENTIFIER)
if ((token = scanToken(ppToken)) != PpAtomIdentifier)
continue;
atom = ppToken->atom;
if (atom == ifAtom || atom == ifdefAtom || atom == ifndefAtom) {
if (atom == PpAtomIf || atom == PpAtomIfdef || atom == PpAtomIfndef) {
depth++;
ifdepth++;
elsetracker++;
} else if (atom == endifAtom) {
} else if (atom == PpAtomEndif) {
token = extraTokenCheck(atom, ppToken, scanToken(ppToken));
elseSeen[elsetracker] = false;
--elsetracker;
@ -304,12 +278,12 @@ int TPpContext::CPPelse(int matchelse, TPpToken* ppToken)
--depth;
--ifdepth;
} else if (matchelse && depth == 0) {
if (atom == elseAtom) {
if (atom == PpAtomElse) {
elseSeen[elsetracker] = true;
token = extraTokenCheck(atom, ppToken, scanToken(ppToken));
// found the #else we are looking for
break;
} else if (atom == elifAtom) {
} else if (atom == PpAtomElif) {
if (elseSeen[elsetracker])
parseContext.ppError(ppToken->loc, "#elif after #else", "#elif", "");
/* we decrement ifdepth here, because CPPif will increment
@ -322,13 +296,13 @@ int TPpContext::CPPelse(int matchelse, TPpToken* ppToken)
return CPPif(ppToken);
}
} else if (atom == elseAtom) {
} else if (atom == PpAtomElse) {
if (elseSeen[elsetracker])
parseContext.ppError(ppToken->loc, "#else after #else", "#else", "");
else
elseSeen[elsetracker] = true;
token = extraTokenCheck(atom, ppToken, scanToken(ppToken));
} else if (atom == elifAtom) {
} else if (atom == PpAtomElif) {
if (elseSeen[elsetracker])
parseContext.ppError(ppToken->loc, "#elif after #else", "#elif", "");
}
@ -344,15 +318,15 @@ int TPpContext::extraTokenCheck(int atom, TPpToken* ppToken, int token)
static const char* message = "unexpected tokens following directive";
const char* label;
if (atom == elseAtom)
if (atom == PpAtomElse)
label = "#else";
else if (atom == elifAtom)
else if (atom == PpAtomElif)
label = "#elif";
else if (atom == endifAtom)
else if (atom == PpAtomEndif)
label = "#endif";
else if (atom == ifAtom)
else if (atom == PpAtomIf)
label = "#if";
else if (atom == lineAtom)
else if (atom == PpAtomLine)
label = "#line";
else
label = "";
@ -405,19 +379,19 @@ namespace {
struct TBinop {
int token, precedence, (*op)(int, int);
} binop[] = {
{ CPP_OR_OP, LOGOR, op_logor },
{ CPP_AND_OP, LOGAND, op_logand },
{ PpAtomOr, LOGOR, op_logor },
{ PpAtomAnd, LOGAND, op_logand },
{ '|', OR, op_or },
{ '^', XOR, op_xor },
{ '&', AND, op_and },
{ CPP_EQ_OP, EQUAL, op_eq },
{ CPP_NE_OP, EQUAL, op_ne },
{ PpAtomEQ, EQUAL, op_eq },
{ PpAtomNE, EQUAL, op_ne },
{ '>', RELATION, op_gt },
{ CPP_GE_OP, RELATION, op_ge },
{ PpAtomGE, RELATION, op_ge },
{ '<', RELATION, op_lt },
{ CPP_LE_OP, RELATION, op_le },
{ CPP_LEFT_OP, SHIFT, op_shl },
{ CPP_RIGHT_OP, SHIFT, op_shr },
{ PpAtomLE, RELATION, op_le },
{ PpAtomLeft, SHIFT, op_shl },
{ PpAtomRight, SHIFT, op_shr },
{ '+', ADD, op_add },
{ '-', ADD, op_sub },
{ '*', MUL, op_mul },
@ -439,15 +413,15 @@ struct TUnop {
int TPpContext::eval(int token, int precedence, bool shortCircuit, int& res, bool& err, TPpToken* ppToken)
{
TSourceLoc loc = ppToken->loc; // because we sometimes read the newline before reporting the error
if (token == CPP_IDENTIFIER) {
if (ppToken->atom == definedAtom) {
if (token == PpAtomIdentifier) {
if (ppToken->atom == PpAtomDefined) {
bool needclose = 0;
token = scanToken(ppToken);
if (token == '(') {
needclose = true;
token = scanToken(ppToken);
}
if (token != CPP_IDENTIFIER) {
if (token != PpAtomIdentifier) {
parseContext.ppError(loc, "incorrect directive, expected identifier", "preprocessor evaluation", "");
err = true;
res = 0;
@ -471,7 +445,7 @@ int TPpContext::eval(int token, int precedence, bool shortCircuit, int& res, boo
token = evalToToken(token, shortCircuit, res, err, ppToken);
return eval(token, precedence, shortCircuit, res, err, ppToken);
}
} else if (token == CPP_INTCONSTANT) {
} else if (token == PpAtomConstInt) {
res = ppToken->ival;
token = scanToken(ppToken);
} else if (token == '(') {
@ -524,8 +498,8 @@ int TPpContext::eval(int token, int precedence, bool shortCircuit, int& res, boo
// Setup short-circuiting, needed for ES, unless already in a short circuit.
// (Once in a short-circuit, can't turn off again, until that whole subexpression is done.
if (! shortCircuit) {
if ((token == CPP_OR_OP && leftSide == 1) ||
(token == CPP_AND_OP && leftSide == 0))
if ((token == PpAtomOr && leftSide == 1) ||
(token == PpAtomAnd && leftSide == 0))
shortCircuit = true;
}
@ -540,7 +514,7 @@ int TPpContext::eval(int token, int precedence, bool shortCircuit, int& res, boo
// Expand macros, skipping empty expansions, to get to the first real token in those expansions.
int TPpContext::evalToToken(int token, bool shortCircuit, int& res, bool& err, TPpToken* ppToken)
{
while (token == CPP_IDENTIFIER && ppToken->atom != definedAtom) {
while (token == PpAtomIdentifier && ppToken->atom != PpAtomDefined) {
int macroReturn = MacroExpand(ppToken->atom, ppToken, true, false);
if (macroReturn == 0) {
parseContext.ppError(ppToken->loc, "can't evaluate expression", "preprocessor evaluation", "");
@ -552,11 +526,10 @@ int TPpContext::evalToToken(int token, bool shortCircuit, int& res, bool& err, T
if (macroReturn == -1) {
if (! shortCircuit && parseContext.profile == EEsProfile) {
const char* message = "undefined macro in expression not allowed in es profile";
const char* name = GetAtomString(ppToken->atom);
if (parseContext.relaxedErrors())
parseContext.ppWarn(ppToken->loc, message, "preprocessor evaluation", name);
parseContext.ppWarn(ppToken->loc, message, "preprocessor evaluation", ppToken->name);
else
parseContext.ppError(ppToken->loc, message, "preprocessor evaluation", name);
parseContext.ppError(ppToken->loc, message, "preprocessor evaluation", ppToken->name);
}
}
token = scanToken(ppToken);
@ -579,7 +552,7 @@ int TPpContext::CPPif(TPpToken* ppToken)
int res = 0;
bool err = false;
token = eval(token, MIN_PRECEDENCE, false, res, err, ppToken);
token = extraTokenCheck(ifAtom, ppToken, token);
token = extraTokenCheck(PpAtomIf, ppToken, token);
if (!res && !err)
token = CPPelse(1, ppToken);
@ -596,7 +569,7 @@ int TPpContext::CPPifdef(int defined, TPpToken* ppToken)
return 0;
}
elsetracker++;
if (token != CPP_IDENTIFIER) {
if (token != PpAtomIdentifier) {
if (defined)
parseContext.ppError(ppToken->loc, "must be followed by macro name", "#ifdef", "");
else
@ -656,7 +629,7 @@ int TPpContext::CPPline(TPpToken* ppToken)
if (!fileErr && !lineErr) {
parseContext.notifyLineDirective(directiveLoc, lineToken, hasFile, fileRes);
}
token = extraTokenCheck(lineAtom, ppToken, token);
token = extraTokenCheck(PpAtomLine, ppToken, token);
return token;
}
@ -669,11 +642,11 @@ int TPpContext::CPPerror(TPpToken* ppToken)
TSourceLoc loc = ppToken->loc;
while (token != '\n') {
if (token == CPP_INTCONSTANT || token == CPP_UINTCONSTANT ||
token == CPP_FLOATCONSTANT || token == CPP_DOUBLECONSTANT) {
if (token == PpAtomConstInt || token == PpAtomConstUint ||
token == PpAtomConstFloat || token == PpAtomConstDouble) {
message.append(ppToken->name);
} else if (token == CPP_IDENTIFIER || token == CPP_STRCONSTANT) {
message.append(GetAtomString(ppToken->atom));
} else if (token == PpAtomIdentifier || token == PpAtomConstString) {
message.append(ppToken->name);
} else {
message.append(GetAtomString(token));
}
@ -691,23 +664,18 @@ int TPpContext::CPPerror(TPpToken* ppToken)
int TPpContext::CPPpragma(TPpToken* ppToken)
{
char SrcStrName[2];
const char* SrcStr;
TVector<TString> tokens;
TSourceLoc loc = ppToken->loc; // because we go to the next line before processing
int token = scanToken(ppToken);
while (token != '\n' && token != EOF) {
switch (token) {
case CPP_IDENTIFIER:
SrcStr = GetAtomString(ppToken->atom);
tokens.push_back(SrcStr);
break;
case CPP_INTCONSTANT:
case CPP_UINTCONSTANT:
case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT:
SrcStr = ppToken->name;
tokens.push_back(SrcStr);
case PpAtomIdentifier:
case PpAtomConstInt:
case PpAtomConstUint:
case PpAtomConstFloat:
case PpAtomConstDouble:
tokens.push_back(ppToken->name);
break;
default:
SrcStrName[0] = (char)token;
@ -740,7 +708,7 @@ int TPpContext::CPPversion(TPpToken* ppToken)
return token;
}
if (token != CPP_INTCONSTANT)
if (token != PpAtomConstInt)
parseContext.ppError(ppToken->loc, "must be followed by version number", "#version", "");
ppToken->ival = atoi(ppToken->name);
@ -752,11 +720,11 @@ int TPpContext::CPPversion(TPpToken* ppToken)
parseContext.notifyVersion(line, versionNumber, nullptr);
return token;
} else {
if (ppToken->atom != coreAtom &&
ppToken->atom != compatibilityAtom &&
ppToken->atom != esAtom)
if (ppToken->atom != PpAtomCore &&
ppToken->atom != PpAtomCompatibility &&
ppToken->atom != PpAtomEs)
parseContext.ppError(ppToken->loc, "bad profile name; use es, core, or compatibility", "#version", "");
parseContext.notifyVersion(line, versionNumber, GetAtomString(ppToken->atom));
parseContext.notifyVersion(line, versionNumber, ppToken->name);
token = scanToken(ppToken);
if (token == '\n')
@ -780,10 +748,10 @@ int TPpContext::CPPextension(TPpToken* ppToken)
return token;
}
if (token != CPP_IDENTIFIER)
if (token != PpAtomIdentifier)
parseContext.ppError(ppToken->loc, "extension name expected", "#extension", "");
strcpy(extensionName, GetAtomString(ppToken->atom));
strcpy(extensionName, ppToken->name);
token = scanToken(ppToken);
if (token != ':') {
@ -792,12 +760,12 @@ int TPpContext::CPPextension(TPpToken* ppToken)
}
token = scanToken(ppToken);
if (token != CPP_IDENTIFIER) {
if (token != PpAtomIdentifier) {
parseContext.ppError(ppToken->loc, "behavior for extension not specified", "#extension", "");
return token;
}
parseContext.updateExtensionBehavior(line, extensionName, GetAtomString(ppToken->atom));
parseContext.updateExtensionBehavior(line, extensionName, ppToken->name);
token = scanToken(ppToken);
if (token == '\n')
@ -812,18 +780,21 @@ int TPpContext::readCPPline(TPpToken* ppToken)
{
int token = scanToken(ppToken);
if (token == CPP_IDENTIFIER) {
if (ppToken->atom == defineAtom) {
if (token == PpAtomIdentifier) {
switch (ppToken->atom) {
case PpAtomDefine:
token = CPPdefine(ppToken);
} else if (ppToken->atom == elseAtom) {
break;
case PpAtomElse:
if (elsetracker[elseSeen])
parseContext.ppError(ppToken->loc, "#else after #else", "#else", "");
elsetracker[elseSeen] = true;
if (! ifdepth)
parseContext.ppError(ppToken->loc, "mismatched statements", "#else", "");
token = extraTokenCheck(elseAtom, ppToken, scanToken(ppToken));
token = extraTokenCheck(PpAtomElse, ppToken, scanToken(ppToken));
token = CPPelse(0, ppToken);
} else if (ppToken->atom == elifAtom) {
break;
case PpAtomElif:
if (! ifdepth)
parseContext.ppError(ppToken->loc, "mismatched statements", "#elif", "");
if (elseSeen[elsetracker])
@ -833,34 +804,46 @@ int TPpContext::readCPPline(TPpToken* ppToken)
while (token != '\n')
token = scanToken(ppToken);
token = CPPelse(0, ppToken);
} else if (ppToken->atom == endifAtom) {
break;
case PpAtomEndif:
elseSeen[elsetracker] = false;
--elsetracker;
if (! ifdepth)
parseContext.ppError(ppToken->loc, "mismatched statements", "#endif", "");
else
--ifdepth;
token = extraTokenCheck(endifAtom, ppToken, scanToken(ppToken));
} else if (ppToken->atom == ifAtom) {
token = CPPif (ppToken);
} else if (ppToken->atom == ifdefAtom) {
token = extraTokenCheck(PpAtomEndif, ppToken, scanToken(ppToken));
break;
case PpAtomIf:
token = CPPif(ppToken);
break;
case PpAtomIfdef:
token = CPPifdef(1, ppToken);
} else if (ppToken->atom == ifndefAtom) {
break;
case PpAtomIfndef:
token = CPPifdef(0, ppToken);
} else if (ppToken->atom == lineAtom) {
break;
case PpAtomLine:
token = CPPline(ppToken);
} else if (ppToken->atom == pragmaAtom) {
break;
case PpAtomPragma:
token = CPPpragma(ppToken);
} else if (ppToken->atom == undefAtom) {
break;
case PpAtomUndef:
token = CPPundef(ppToken);
} else if (ppToken->atom == errorAtom) {
break;
case PpAtomError:
token = CPPerror(ppToken);
} else if (ppToken->atom == versionAtom) {
break;
case PpAtomVersion:
token = CPPversion(ppToken);
} else if (ppToken->atom == extensionAtom) {
break;
case PpAtomExtension:
token = CPPextension(ppToken);
} else {
parseContext.ppError(ppToken->loc, "invalid directive:", "#", GetAtomString(ppToken->atom));
break;
default:
parseContext.ppError(ppToken->loc, "invalid directive:", "#", ppToken->name);
break;
}
} else if (token != '\n' && token != EOF)
parseContext.ppError(ppToken->loc, "invalid directive", "#", "");
@ -878,7 +861,7 @@ TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream* a, TPpToken* p
RewindTokenStream(a);
do {
token = ReadToken(a, ppToken);
if (token == CPP_IDENTIFIER && LookUpSymbol(ppToken->atom))
if (token == PpAtomIdentifier && LookUpSymbol(ppToken->atom))
break;
} while (token != tInput::endOfInput);
@ -889,7 +872,7 @@ TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream* a, TPpToken* p
pushInput(new tMarkerInput(this));
pushTokenStreamInput(a);
while ((token = scanToken(ppToken)) != tMarkerInput::marker) {
if (token == CPP_IDENTIFIER && MacroExpand(ppToken->atom, ppToken, false, newLineOkay) != 0)
if (token == PpAtomIdentifier && MacroExpand(ppToken->atom, ppToken, false, newLineOkay) != 0)
continue;
RecordToken(n, token, ppToken);
}
@ -910,7 +893,7 @@ int TPpContext::tMacroInput::scan(TPpToken* ppToken)
} while (token == ' '); // handle white space in macro
// TODO: preprocessor: properly handle whitespace (or lack of it) between tokens when expanding
if (token == CPP_IDENTIFIER) {
if (token == PpAtomIdentifier) {
int i;
for (i = mac->argc - 1; i >= 0; i--)
if (mac->args[i] == ppToken->atom)
@ -939,7 +922,7 @@ int TPpContext::tZeroInput::scan(TPpToken* ppToken)
ppToken->space = false;
done = true;
return CPP_INTCONSTANT;
return PpAtomConstInt;
}
//
@ -952,35 +935,34 @@ int TPpContext::tZeroInput::scan(TPpToken* ppToken)
//
int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef, bool newLineOkay)
{
ppToken->space = false;
switch (atom) {
case PpAtomLineMacro:
ppToken->ival = parseContext.getCurrentLoc().line;
sprintf(ppToken->name, "%d", ppToken->ival);
UngetToken(PpAtomConstInt, ppToken);
return 1;
case PpAtomFileMacro:
ppToken->ival = parseContext.getCurrentLoc().string;
sprintf(ppToken->name, "%d", ppToken->ival);
UngetToken(PpAtomConstInt, ppToken);
return 1;
case PpAtomVersionMacro:
ppToken->ival = parseContext.version;
sprintf(ppToken->name, "%d", ppToken->ival);
UngetToken(PpAtomConstInt, ppToken);
return 1;
default:
break;
}
Symbol *sym = LookUpSymbol(atom);
int token;
int depth = 0;
ppToken->space = false;
if (atom == __LINE__Atom) {
ppToken->ival = parseContext.getCurrentLoc().line;
sprintf(ppToken->name, "%d", ppToken->ival);
UngetToken(CPP_INTCONSTANT, ppToken);
return 1;
}
if (atom == __FILE__Atom) {
ppToken->ival = parseContext.getCurrentLoc().string;
sprintf(ppToken->name, "%d", ppToken->ival);
UngetToken(CPP_INTCONSTANT, ppToken);
return 1;
}
if (atom == __VERSION__Atom) {
ppToken->ival = parseContext.version;
sprintf(ppToken->name, "%d", ppToken->ival);
UngetToken(CPP_INTCONSTANT, ppToken);
return 1;
}
// no recursive expansions
if (sym && sym->mac.busy)
return 0;

56
glslang/MachineIndependent/preprocessor/PpAtom.cpp
View file

@ -98,27 +98,28 @@ const struct {
int val;
const char* str;
} tokens[] = {
{ CPP_AND_OP, "&&" },
{ CPP_AND_ASSIGN, "&=" },
{ CPP_SUB_ASSIGN, "-=" },
{ CPP_MOD_ASSIGN, "%=" },
{ CPP_ADD_ASSIGN, "+=" },
{ CPP_DIV_ASSIGN, "/=" },
{ CPP_MUL_ASSIGN, "*=" },
{ CPP_EQ_OP, "==" },
{ CPP_XOR_OP, "^^" },
{ CPP_XOR_ASSIGN, "^=" },
{ CPP_GE_OP, ">=" },
{ CPP_RIGHT_OP, ">>" },
{ CPP_RIGHT_ASSIGN, ">>="},
{ CPP_LE_OP, "<=" },
{ CPP_LEFT_OP, "<<" },
{ CPP_LEFT_ASSIGN, "<<="},
{ CPP_DEC_OP, "--" },
{ CPP_NE_OP, "!=" },
{ CPP_OR_OP, "||" },
{ CPP_OR_ASSIGN, "|=" },
{ CPP_INC_OP, "++" },
{ PpAtomDefine, "define" },
{ PpAtomDefined, "defined" },
{ PpAtomUndef, "undef" },
{ PpAtomIf, "if" },
{ PpAtomElif, "elif" },
{ PpAtomElse, "else" },
{ PpAtomEndif, "endif" },
{ PpAtomIfdef, "ifdef" },
{ PpAtomIfndef, "ifndef" },
{ PpAtomLine, "line" },
{ PpAtomPragma, "pragma" },
{ PpAtomError, "error" },
{ PpAtomVersion, "version" },
{ PpAtomCore, "core" },
{ PpAtomCompatibility, "compatibility" },
{ PpAtomEs, "es" },
{ PpAtomExtension, "extension" },
{ PpAtomLineMacro, "__LINE__" },
{ PpAtomFileMacro, "__FILE__" },
{ PpAtomVersionMacro, "__VERSION__" },
};
} // end anonymous namespace
@ -131,9 +132,10 @@ namespace glslang {
int TPpContext::LookUpAddString(const char* s)
{
auto it = atomMap.find(s);
if (it == atomMap.end())
return AddAtomFixed(s, nextAtom++);
else
if (it == atomMap.end()) {
AddAtomFixed(s, nextAtom);
return nextAtom++;
} else
return it->second;
}
@ -159,14 +161,12 @@ const char* TPpContext::GetAtomString(int atom)
//
// Add forced mapping of string to atom.
//
int TPpContext::AddAtomFixed(const char* s, int atom)
void TPpContext::AddAtomFixed(const char* s, int atom)
{
auto it = atomMap.insert(std::pair<TString, int>(s, atom)).first;
if (stringMap.size() < (size_t)atom + 1)
stringMap.resize(atom + 100, 0);
stringMap[atom] = &it->first;
return atom;
}
//
@ -189,7 +189,7 @@ void TPpContext::InitAtomTable()
for (int ii = 0; ii < sizeof(tokens)/sizeof(tokens[0]); ii++)
AddAtomFixed(tokens[ii].str, tokens[ii].val);
nextAtom = CPP_FIRST_USER_TOKEN_SY;
nextAtom = PpAtomLast;
}
} // end namespace glslang

42
glslang/MachineIndependent/preprocessor/PpContext.cpp
View file

@ -93,48 +93,6 @@ TPpContext::TPpContext(TParseContext& pc) :
for (elsetracker = 0; elsetracker < maxIfNesting; elsetracker++)
elseSeen[elsetracker] = false;
elsetracker = 0;
// The following identifies all legal characters in GLSL:
//for (int c = 0; c < 256; ++c)
// languageCharacters[c] = false;
//for (int c = 'a'; c <= 'z'; ++c)
// languageCharacters[c] = true;
//for (int c = 'A'; c <= 'Z'; ++c)
// languageCharacters[c] = true;
//languageCharacters['_'] = true;
//for (int c = '0'; c <= '9'; ++c)
// languageCharacters[c] = true;
//languageCharacters['.'] = true;
//languageCharacters['+'] = true;
//languageCharacters['-'] = true;
//languageCharacters['/'] = true;
//languageCharacters['*'] = true;
//languageCharacters['%'] = true;
//languageCharacters['<'] = true;
//languageCharacters['>'] = true;
//languageCharacters['['] = true;
//languageCharacters[']'] = true;
//languageCharacters['('] = true;
//languageCharacters[')'] = true;
//languageCharacters['{'] = true;
//languageCharacters['}'] = true;
//languageCharacters['^'] = true;
//languageCharacters['|'] = true;
//languageCharacters['&'] = true;
//languageCharacters['~'] = true;
//languageCharacters['='] = true;
//languageCharacters['!'] = true;
//languageCharacters[':'] = true;
//languageCharacters[';'] = true;
//languageCharacters[','] = true;
//languageCharacters['?'] = true;
//languageCharacters['#'] = true;
//// white space
//languageCharacters[' '] = true;
//for (int c = 9; c <= 13; ++c)
// languageCharacters[c] = true;
}
TPpContext::~TPpContext()

27
glslang/MachineIndependent/preprocessor/PpContext.h
View file

@ -287,31 +287,6 @@ protected:
//
// from Pp.cpp
//
int bindAtom;
int constAtom;
int defaultAtom;
int defineAtom;
int definedAtom;
int elseAtom;
int elifAtom;
int endifAtom;
int ifAtom;
int ifdefAtom;
int ifndefAtom;
int includeAtom;
int lineAtom;
int pragmaAtom;
int texunitAtom;
int undefAtom;
int errorAtom;
int __LINE__Atom;
int __FILE__Atom;
int __VERSION__Atom;
int versionAtom;
int coreAtom;
int compatibilityAtom;
int esAtom;
int extensionAtom;
TSourceLoc ifloc; /* outermost #if */
int InitCPP();
@ -467,7 +442,7 @@ protected:
TStringMap stringMap;
int nextAtom;
void InitAtomTable();
int AddAtomFixed(const char* s, int atom);
void AddAtomFixed(const char* s, int atom);
int LookUpAddString(const char* s);
const char* GetAtomString(int atom);

155
glslang/MachineIndependent/preprocessor/PpScanner.cpp
View file

@ -223,9 +223,9 @@ int TPpContext::lFloatConst(int len, int ch, TPpToken* ppToken)
}
if (isDouble)
return CPP_DOUBLECONSTANT;
return PpAtomConstDouble;
else
return CPP_FLOATCONSTANT;
return PpAtomConstFloat;
}
//
@ -233,7 +233,7 @@ int TPpContext::lFloatConst(int len, int ch, TPpToken* ppToken)
//
int TPpContext::tStringInput::scan(TPpToken* ppToken)
{
char tokenText[TPpToken::maxTokenLength + 1];
char* tokenText = ppToken->name;
int AlreadyComplained = 0;
int len = 0;
int ch = 0;
@ -253,7 +253,8 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
len = 0;
switch (ch) {
default:
return ch; // Single character token, including '#' and '\' (escaped newlines are handled at a lower level, so this is just a '\' token)
// Single character token, including '#' and '\' (escaped newlines are handled at a lower level, so this is just a '\' token)
return ch;
case EOF:
return endOfInput;
@ -273,7 +274,7 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
do {
if (len < TPpToken::maxTokenLength) {
tokenText[len++] = (char)ch;
ch = pp->getChar();
ch = pp->getChar();
} else {
if (! AlreadyComplained) {
pp->parseContext.ppError(ppToken->loc, "name too long", "", "");
@ -293,8 +294,7 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
tokenText[len] = '\0';
pp->ungetChar();
ppToken->atom = pp->LookUpAddString(tokenText);
return CPP_IDENTIFIER;
return PpAtomIdentifier;
case '0':
ppToken->name[len++] = (char)ch;
ch = pp->getChar();
@ -345,9 +345,9 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
ppToken->ival = (int)ival;
if (isUnsigned)
return CPP_UINTCONSTANT;
return PpAtomConstUint;
else
return CPP_INTCONSTANT;
return PpAtomConstInt;
} else {
// could be octal integer or floating point, speculative pursue octal until it must be floating point
@ -406,9 +406,9 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
ppToken->ival = (int)ival;
if (isUnsigned)
return CPP_UINTCONSTANT;
return PpAtomConstUint;
else
return CPP_INTCONSTANT;
return PpAtomConstInt;
}
break;
case '1': case '2': case '3': case '4':
@ -453,17 +453,17 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
ppToken->ival = (int)ival;
if (uint)
return CPP_UINTCONSTANT;
return PpAtomConstUint;
else
return CPP_INTCONSTANT;
return PpAtomConstInt;
}
break;
case '-':
ch = pp->getChar();
if (ch == '-') {
return CPP_DEC_OP;
return PpAtomDecrement;
} else if (ch == '=') {
return CPP_SUB_ASSIGN;
return PpAtomSub;
} else {
pp->ungetChar();
return '-';
@ -471,9 +471,9 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
case '+':
ch = pp->getChar();
if (ch == '+') {
return CPP_INC_OP;
return PpAtomIncrement;
} else if (ch == '=') {
return CPP_ADD_ASSIGN;
return PpAtomAdd;
} else {
pp->ungetChar();
return '+';
@ -481,7 +481,7 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
case '*':
ch = pp->getChar();
if (ch == '=') {
return CPP_MUL_ASSIGN;
return PpAtomMul;
} else {
pp->ungetChar();
return '*';
@ -489,28 +489,18 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
case '%':
ch = pp->getChar();
if (ch == '=') {
return CPP_MOD_ASSIGN;
} else if (ch == '>'){
return CPP_RIGHT_BRACE;
return PpAtomMod;
} else {
pp->ungetChar();
return '%';
}
case ':':
ch = pp->getChar();
if (ch == '>') {
return CPP_RIGHT_BRACKET;
} else {
pp->ungetChar();
return ':';
}
case '^':
ch = pp->getChar();
if (ch == '^') {
return CPP_XOR_OP;
return PpAtomXor;
} else {
if (ch == '=')
return CPP_XOR_ASSIGN;
return PpAtomXorAssign;
else{
pp->ungetChar();
return '^';
@ -520,7 +510,7 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
case '=':
ch = pp->getChar();
if (ch == '=') {
return CPP_EQ_OP;
return PpAtomEQ;
} else {
pp->ungetChar();
return '=';
@ -528,7 +518,7 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
case '!':
ch = pp->getChar();
if (ch == '=') {
return CPP_NE_OP;
return PpAtomNE;
} else {
pp->ungetChar();
return '!';
@ -536,68 +526,54 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
case '|':
ch = pp->getChar();
if (ch == '|') {
return CPP_OR_OP;
return PpAtomOr;
} else if (ch == '=') {
return PpAtomOrAssign;
} else {
if (ch == '=')
return CPP_OR_ASSIGN;
else{
pp->ungetChar();
return '|';
}
pp->ungetChar();
return '|';
}
case '&':
ch = pp->getChar();
if (ch == '&') {
return CPP_AND_OP;
return PpAtomAnd;
} else if (ch == '=') {
return PpAtomAndAssign;
} else {
if (ch == '=')
return CPP_AND_ASSIGN;
else{
pp->ungetChar();
return '&';
}
pp->ungetChar();
return '&';
}
case '<':
ch = pp->getChar();
if (ch == '<') {
ch = pp->getChar();
if (ch == '=')
return CPP_LEFT_ASSIGN;
else{
return PpAtomLeftAssign;
else {
pp->ungetChar();
return CPP_LEFT_OP;
return PpAtomLeft;
}
} else if (ch == '=') {
return PpAtomLE;
} else {
if (ch == '=') {
return CPP_LE_OP;
} else {
if (ch == '%')
return CPP_LEFT_BRACE;
else if (ch == ':')
return CPP_LEFT_BRACKET;
else{
pp->ungetChar();
return '<';
}
}
pp->ungetChar();
return '<';
}
case '>':
ch = pp->getChar();
if (ch == '>') {
ch = pp->getChar();
if (ch == '=')
return CPP_RIGHT_ASSIGN;
else{
return PpAtomRightAssign;
else {
pp->ungetChar();
return CPP_RIGHT_OP;
return PpAtomRight;
}
} else if (ch == '=') {
return PpAtomGE;
} else {
if (ch == '=') {
return CPP_GE_OP;
} else {
pp->ungetChar();
return '>';
}
pp->ungetChar();
return '>';
}
case '.':
ch = pp->getChar();
@ -642,7 +618,7 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
// loop again to get the next token...
break;
} else if (ch == '=') {
return CPP_DIV_ASSIGN;
return PpAtomDiv;
} else {
pp->ungetChar();
return '/';
@ -659,13 +635,11 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
break;
};
tokenText[len] = '\0';
if (ch == '"') {
ppToken->atom = pp->LookUpAddString(tokenText);
return CPP_STRCONSTANT;
} else {
if (ch != '"') {
pp->ungetChar();
pp->parseContext.ppError(ppToken->loc, "end of line in string", "string", "");
return CPP_ERROR_SY;
}
return PpAtomConstString;
}
ch = pp->getChar();
@ -684,7 +658,6 @@ const char* TPpContext::tokenize(TPpToken* ppToken)
int token = '\n';
for(;;) {
const char* tokenString = nullptr;
token = scanToken(ppToken);
ppToken->token = token;
if (token == EOF) {
@ -710,22 +683,28 @@ const char* TPpContext::tokenize(TPpToken* ppToken)
continue;
// expand macros
if (token == CPP_IDENTIFIER && MacroExpand(ppToken->atom, ppToken, false, true) != 0)
if (token == PpAtomIdentifier && MacroExpand(ppToken->atom, ppToken, false, true) != 0)
continue;
if (token == CPP_IDENTIFIER)
tokenString = GetAtomString(ppToken->atom);
else if (token == CPP_INTCONSTANT || token == CPP_UINTCONSTANT ||
token == CPP_FLOATCONSTANT || token == CPP_DOUBLECONSTANT)
const char* tokenString = nullptr;
switch (token) {
case PpAtomIdentifier:
case PpAtomConstInt:
case PpAtomConstUint:
case PpAtomConstFloat:
case PpAtomConstDouble:
tokenString = ppToken->name;
else if (token == CPP_STRCONSTANT) {
break;
case PpAtomConstString:
parseContext.ppError(ppToken->loc, "string literals not supported", "\"\"", "");
tokenString = nullptr;
} else if (token == '\'') {
break;
case '\'':
parseContext.ppError(ppToken->loc, "character literals not supported", "\'", "");
tokenString = nullptr;
} else
break;
default:
tokenString = GetAtomString(token);
break;
}
if (tokenString) {
if (tokenString[0] != 0)

47
glslang/MachineIndependent/preprocessor/PpTokens.cpp
View file

@ -126,25 +126,25 @@ void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken* ppToken)
const char* s;
char* str = NULL;
if (token > 256)
if (token > PpAtomMaxSingle)
lAddByte(pTok, (unsigned char)((token & 0x7f) + 0x80));
else
lAddByte(pTok, (unsigned char)(token & 0x7f));
switch (token) {
case CPP_IDENTIFIER:
case CPP_STRCONSTANT:
s = GetAtomString(ppToken->atom);
case PpAtomIdentifier:
case PpAtomConstString:
s = ppToken->name;
while (*s)
lAddByte(pTok, (unsigned char) *s++);
lAddByte(pTok, 0);
break;
case CPP_INTCONSTANT:
case CPP_UINTCONSTANT:
case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT:
case PpAtomConstInt:
case PpAtomConstUint:
case PpAtomConstFloat:
case PpAtomConstDouble:
str = ppToken->name;
while (*str){
while (*str) {
lAddByte(pTok, (unsigned char) *str);
str++;
}
@ -168,7 +168,7 @@ void TPpContext::RewindTokenStream(TokenStream *pTok)
*/
int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
{
char tokenText[TPpToken::maxTokenLength + 1];
char* tokenText = ppToken->name;
int ltoken, len;
int ch;
@ -182,17 +182,17 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
parseContext.requireProfile(ppToken->loc, ~EEsProfile, "token pasting (##)");
parseContext.profileRequires(ppToken->loc, ~EEsProfile, 130, 0, "token pasting (##)");
parseContext.error(ppToken->loc, "token pasting not implemented (internal error)", "##", "");
//return CPP_TOKEN_PASTE;
//return PpAtomPaste;
return ReadToken(pTok, ppToken);
} else
lUnreadByte(pTok);
break;
case CPP_STRCONSTANT:
case CPP_IDENTIFIER:
case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT:
case CPP_INTCONSTANT:
case CPP_UINTCONSTANT:
case PpAtomConstString:
case PpAtomIdentifier:
case PpAtomConstFloat:
case PpAtomConstDouble:
case PpAtomConstInt:
case PpAtomConstUint:
len = 0;
ch = lReadByte(pTok);
while (ch != 0) {
@ -208,17 +208,18 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
tokenText[len] = 0;
switch (ltoken) {
case CPP_IDENTIFIER:
case CPP_STRCONSTANT:
case PpAtomIdentifier:
ppToken->atom = LookUpAddString(tokenText);
break;
case CPP_FLOATCONSTANT:
case CPP_DOUBLECONSTANT:
case PpAtomConstString:
break;
case PpAtomConstFloat:
case PpAtomConstDouble:
strcpy(ppToken->name, tokenText);
ppToken->dval = atof(ppToken->name);
break;
case CPP_INTCONSTANT:
case CPP_UINTCONSTANT:
case PpAtomConstInt:
case PpAtomConstUint:
strcpy(ppToken->name, tokenText);
if (len > 0 && tokenText[0] == '0') {
if (len > 1 && (tokenText[1] == 'x' || tokenText[1] == 'X'))

117
glslang/MachineIndependent/preprocessor/PpTokens.h
View file

@ -78,40 +78,89 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef PARSER_H
#define PARSER_H
#define CPP_AND_OP 257
#define CPP_SUB_ASSIGN 259
#define CPP_MOD_ASSIGN 260
#define CPP_ADD_ASSIGN 261
#define CPP_DIV_ASSIGN 262
#define CPP_MUL_ASSIGN 263
#define CPP_EQ_OP 264
#define CPP_XOR_OP 265
#define CPP_ERROR_SY 266
#define CPP_FLOATCONSTANT 267
#define CPP_GE_OP 268
#define CPP_RIGHT_OP 269
#define CPP_IDENTIFIER 270
#define CPP_INTCONSTANT 271
#define CPP_LE_OP 272
#define CPP_LEFT_OP 273
#define CPP_DEC_OP 274
#define CPP_NE_OP 275
#define CPP_OR_OP 276
#define CPP_INC_OP 277
#define CPP_STRCONSTANT 278
#define CPP_RIGHT_ASSIGN 280
#define CPP_LEFT_ASSIGN 281
#define CPP_AND_ASSIGN 282
#define CPP_OR_ASSIGN 283
#define CPP_XOR_ASSIGN 284
#define CPP_LEFT_BRACKET 285
#define CPP_RIGHT_BRACKET 286
#define CPP_LEFT_BRACE 287
#define CPP_RIGHT_BRACE 288
#define CPP_UINTCONSTANT 289
#define CPP_DOUBLECONSTANT 290
#define CPP_TOKEN_PASTE 291
namespace glslang {
#define CPP_FIRST_USER_TOKEN_SY 292
// Multi-character tokens
enum EFixedAtoms {
PpAtomMaxSingle = 256, // single character tokens get their own char value as their token, skip them
// Operators
PpAtomAdd,
PpAtomSub,
PpAtomMul,
PpAtomDiv,
PpAtomMod,
PpAtomRight,
PpAtomLeft,
PpAtomRightAssign,
PpAtomLeftAssign,
PpAtomAndAssign,
PpAtomOrAssign,
PpAtomXorAssign,
PpAtomAnd,
PpAtomOr,
PpAtomXor,
PpAtomEQ,
PpAtomNE,
PpAtomGE,
PpAtomLE,
PpAtomDecrement,
PpAtomIncrement,
PpAtomPaste,
// Constants
PpAtomConstInt,
PpAtomConstUint,
PpAtomConstFloat,
PpAtomConstDouble,
PpAtomConstString,
// Indentifiers
PpAtomIdentifier,
// preprocessor "keywords"
PpAtomDefine,
PpAtomDefined,
PpAtomUndef,
PpAtomIf,
PpAtomIfdef,
PpAtomIfndef,
PpAtomElse,
PpAtomElif,
PpAtomEndif,
PpAtomLine,
PpAtomPragma,
PpAtomError,
// #version ...
PpAtomVersion,
PpAtomCore,
PpAtomCompatibility,
PpAtomEs,
// #extension
PpAtomExtension,
// __LINE__, __FILE__, __VERSION__
PpAtomLineMacro,
PpAtomFileMacro,
PpAtomVersionMacro,
PpAtomLast,
};
} // end namespace glslang
#endif /* not PARSER_H */