Improve multi-threading and move Standalone to a multi-threading model (currently off though).

git-svn-id: https://cvs.khronos.org/svn/repos/ogl/trunk/ecosystem/public/sdk/tools/glslang@22565 e7fa87d3-cd2b-0410-9028-fcbf551c1848

glslang/Include/Common.h

@@ -92,7 +92,7 @@ typedef pool_allocator<char> TStringAllocator;
 typedef std::basic_string <char, std::char_traits<char>, TStringAllocator > TString;
 inline TString* NewPoolTString(const char* s)
 {
-	void* memory = GlobalPoolAllocator.allocate(sizeof(TString));
+	void* memory = GetThreadPoolAllocator().allocate(sizeof(TString));
 	return new(memory) TString(s);
 }
 

glslang/Include/ConstantUnion.h

@@ -41,7 +41,7 @@
 class constUnion {
 public:
 
-    POOL_ALLOCATOR_NEW_DELETE(GlobalPoolAllocator)
+    POOL_ALLOCATOR_NEW_DELETE(GetThreadPoolAllocator())
     void setIConst(int i)
     { 
         iConst = i; 

glslang/Include/PoolAlloc.h

@@ -144,7 +144,7 @@ private:
 //
 class TPoolAllocator {
 public:
-    TPoolAllocator(bool global = false, int growthIncrement = 8*1024, int allocationAlignment = 16);
+    TPoolAllocator(int growthIncrement = 8*1024, int allocationAlignment = 16);
 
     //
     // Don't call the destructor just to free up the memory, call pop()
@@ -222,7 +222,6 @@ protected:
         return TAllocation::offsetAllocation(memory);
     }
 
-    bool global;            // should be true if this object is globally scoped
     size_t pageSize;        // granularity of allocation from the OS
     size_t alignment;       // all returned allocations will be aligned at 
                             //      this granularity, which will be a power of 2
@@ -249,16 +248,14 @@ private:
 // with everyone using the same global allocator.
 //
 typedef TPoolAllocator* PoolAllocatorPointer;
-extern TPoolAllocator& GetGlobalPoolAllocator();
-#define GlobalPoolAllocator GetGlobalPoolAllocator()
-
+extern TPoolAllocator& GetThreadPoolAllocator();
 
 struct TThreadGlobalPools
 {
         TPoolAllocator* globalPoolAllocator;
 };
 
-void SetGlobalPoolAllocatorPtr(TPoolAllocator& poolAllocator);
+void SetThreadPoolAllocator(TPoolAllocator& poolAllocator);
 
 //
 // This STL compatible allocator is intended to be used as the allocator
@@ -284,7 +281,7 @@ public:
     pointer address(reference x) const { return &x; }
     const_pointer address(const_reference x) const { return &x; }
 
-    pool_allocator() : allocator(GlobalPoolAllocator) { }
+    pool_allocator() : allocator(GetThreadPoolAllocator()) { }
     pool_allocator(TPoolAllocator& a) : allocator(a) { }
     pool_allocator(const pool_allocator<T>& p) : allocator(p.allocator) { }
 

glslang/Include/Types.h

@@ -150,7 +150,7 @@ typedef TVector<TTypeLoc> TTypeList;
 
 inline TTypeList* NewPoolTTypeList()
 {
-	void* memory = GlobalPoolAllocator.allocate(sizeof(TTypeList));
+	void* memory = GetThreadPoolAllocator().allocate(sizeof(TTypeList));
 	return new(memory) TTypeList;
 }
 
@@ -158,7 +158,7 @@ typedef TVector<TString*> TIdentifierList;
 
 inline TIdentifierList* NewPoolTIdentifierList()
 {
-    void* memory = GlobalPoolAllocator.allocate(sizeof(TIdentifierList));
+    void* memory = GetThreadPoolAllocator().allocate(sizeof(TIdentifierList));
     return new(memory) TIdentifierList;
 }
 
@@ -176,7 +176,7 @@ typedef TVector<int>* TArraySizes;
 
 inline TArraySizes NewPoolTArraySizes()
 {
-    void* memory = GlobalPoolAllocator.allocate(sizeof(TVector<int>));
+    void* memory = GetThreadPoolAllocator().allocate(sizeof(TVector<int>));
     return new(memory) TVector<int>;
 }
 
@@ -401,7 +401,7 @@ typedef std::map<TTypeList*, TTypeList*>::const_iterator TStructureMapIterator;
 //
 class TType {
 public:
-    POOL_ALLOCATOR_NEW_DELETE(GlobalPoolAllocator)
+    POOL_ALLOCATOR_NEW_DELETE(GetThreadPoolAllocator())
     explicit TType(TBasicType t, TStorageQualifier q = EvqTemporary, int vs = 1, int mc = 0, int mr = 0) :
                             basicType(t), vectorSize(vs), matrixCols(mc), matrixRows(mr), arraySizes(0),
                             structure(0), structureSize(0), maxArraySize(0), arrayInformationType(0),

glslang/Include/intermediate.h

@@ -318,7 +318,7 @@ class TInfoSink;
 //
 class TIntermNode {
 public:
-    POOL_ALLOCATOR_NEW_DELETE(GlobalPoolAllocator)
+    POOL_ALLOCATOR_NEW_DELETE(GetThreadPoolAllocator())
 
     TIntermNode() { loc.line = 0; loc.string = 0; }
     virtual TSourceLoc getLoc() const { return loc; }
@@ -601,7 +601,7 @@ protected:
 //
 class TIntermTraverser {
 public:
-    POOL_ALLOCATOR_NEW_DELETE(GlobalPoolAllocator)
+    POOL_ALLOCATOR_NEW_DELETE(GetThreadPoolAllocator())
 
     TIntermTraverser() : 
         visitSymbol(0), 

glslang/MachineIndependent/Initialize.h

@@ -47,7 +47,7 @@ typedef TVector<TString> TBuiltInStrings;
 
 class TBuiltIns {
 public:
-    POOL_ALLOCATOR_NEW_DELETE(GlobalPoolAllocator)
+    POOL_ALLOCATOR_NEW_DELETE(GetThreadPoolAllocator())
     TBuiltIns();
     virtual ~TBuiltIns();
     void initialize(int version, EProfile);

glslang/MachineIndependent/PoolAlloc.cpp

@@ -46,7 +46,7 @@ void InitializeGlobalPools()
     if (globalPools)
         return;
 
-    TPoolAllocator *globalPoolAllocator = new TPoolAllocator(true);
+    TPoolAllocator *globalPoolAllocator = new TPoolAllocator();
 
     TThreadGlobalPools* threadData = new TThreadGlobalPools();
     
@@ -58,12 +58,12 @@ void InitializeGlobalPools()
 void FreeGlobalPools()
 {
     // Release the allocated memory for this thread.
-    TThreadGlobalPools* globalPools= static_cast<TThreadGlobalPools*>(OS_GetTLSValue(PoolIndex));    
-    if (!globalPools)
+    TThreadGlobalPools* globalPools = static_cast<TThreadGlobalPools*>(OS_GetTLSValue(PoolIndex));    
+    if (! globalPools)
         return;
 	
-    GlobalPoolAllocator.popAll();
-    delete &GlobalPoolAllocator;       
+    GetThreadPoolAllocator().popAll();
+    delete &GetThreadPoolAllocator();       
     delete globalPools;
 }
 
@@ -82,14 +82,14 @@ void FreePoolIndex()
     OS_FreeTLSIndex(PoolIndex);
 }
 
-TPoolAllocator& GetGlobalPoolAllocator()
+TPoolAllocator& GetThreadPoolAllocator()
 {
     TThreadGlobalPools* threadData = static_cast<TThreadGlobalPools*>(OS_GetTLSValue(PoolIndex));
 
     return *threadData->globalPoolAllocator;
 }
 
-void SetGlobalPoolAllocatorPtr(TPoolAllocator& poolAllocator)
+void SetThreadPoolAllocator(TPoolAllocator& poolAllocator)
 {
     TThreadGlobalPools* threadData = static_cast<TThreadGlobalPools*>(OS_GetTLSValue(PoolIndex));
 
@@ -100,8 +100,7 @@ void SetGlobalPoolAllocatorPtr(TPoolAllocator& poolAllocator)
 // Implement the functionality of the TPoolAllocator class, which
 // is documented in PoolAlloc.h.
 //
-TPoolAllocator::TPoolAllocator(bool g, int growthIncrement, int allocationAlignment) : 
-    global(g),
+TPoolAllocator::TPoolAllocator(int growthIncrement, int allocationAlignment) : 
     pageSize(growthIncrement),
     alignment(allocationAlignment),
     freeList(0),
@@ -148,19 +147,12 @@ TPoolAllocator::TPoolAllocator(bool g, int growthIncrement, int allocationAlignm
 
 TPoolAllocator::~TPoolAllocator()
 {
-    if (!global) {
-        //
-        // Then we know that this object is not being 
-        // allocated after other, globally scoped objects
-        // that depend on it.  So we can delete the "in use" memory.
-        //
-	    while (inUseList) {
-	        tHeader* next = inUseList->nextPage;
-            inUseList->~tHeader();
-            delete [] reinterpret_cast<char*>(inUseList);
-	        inUseList = next;
-	    }
-    }
+	while (inUseList) {
+	    tHeader* next = inUseList->nextPage;
+        inUseList->~tHeader();
+        delete [] reinterpret_cast<char*>(inUseList);
+	    inUseList = next;
+	}
 
     //
     // Always delete the free list memory - it can't be being

glslang/MachineIndependent/ShaderLang.cpp

@@ -79,7 +79,7 @@ int MapVersionToIndex(int version)
 const int VersionCount = 12;
 
 //
-// A symbol table per version per profile per language.  This will be sparsely
+// A process-global symbol table per version per profile per language.  This will be sparsely
 // populated, so they will only only be generated as needed.
 // 
 // Each has a different set of built-ins, and we want to preserve that from
@@ -163,41 +163,59 @@ bool AddContextSpecificSymbols(const TBuiltInResource* resources, TInfoSink& inf
     return true;
 }
 
+//
+// To do this on the fly, we want to leave the current state of our thread's 
+// pool allocator intact, so:
+//  - Switch to a new pool for parsing the built-ins
+//  - Do the parsing, which builds the symbol table, using the new pool
+//  - Switch to the process-global pool to save a copy the resulting symbol table
+//  - Free up the new pool used to parse the built-ins
+//  - Switch back to the original thread's pool
+//
+// This only gets done the first time any thread needs a particular symbol table
+// (lazy evaluation).
+//
 void SetupBuiltinSymbolTable(int version, EProfile profile)
 {
     TInfoSink infoSink;
 
-    // This function is for lazy setup.  See if already done.
+    // Make sure only one thread tries to do this at a time
+    glslang::GetGlobalLock();
+
+    // See if it's already been done.
     int versionIndex = MapVersionToIndex(version);
-    if (SharedSymbolTables[versionIndex][profile][EShLangVertex])
+    if (SharedSymbolTables[versionIndex][profile][EShLangVertex]) {
+        glslang::ReleaseGlobalLock();
+
         return;
+    }
 
-    TPoolAllocator& savedGPA = GetGlobalPoolAllocator();
-    TPoolAllocator *builtInPoolAllocator = new TPoolAllocator(true);
-    SetGlobalPoolAllocatorPtr(*builtInPoolAllocator);
+    // Switch to a new pool
+    TPoolAllocator& savedGPA = GetThreadPoolAllocator();
+    TPoolAllocator* builtInPoolAllocator = new TPoolAllocator();
+    SetThreadPoolAllocator(*builtInPoolAllocator);
 
+    // Generate the symbol table using the new pool
     TSymbolTable symTables[EShLangCount];
     if (profile == EEsProfile) {
         for (int stage = 0; stage < EShLangCount; ++stage)
             symTables[stage].setNoBuiltInRedeclarations();
     }
-
     GenerateBuiltInSymbolTable(infoSink, symTables, version, profile);
 
-    SetGlobalPoolAllocatorPtr(*PerProcessGPA);
+    // Switch to the process-global pool
+    SetThreadPoolAllocator(*PerProcessGPA);
 
+    // Copy the symbol table from the new pool to the process-global pool
     SharedSymbolTables[versionIndex][profile][EShLangVertex] = new TSymbolTable;
     SharedSymbolTables[versionIndex][profile][EShLangVertex]->copyTable(symTables[EShLangVertex]);
     SharedSymbolTables[versionIndex][profile][EShLangFragment] = new TSymbolTable;
     SharedSymbolTables[versionIndex][profile][EShLangFragment]->copyTable(symTables[EShLangFragment]);
-        
-    symTables[EShLangVertex].pop(0);
-    symTables[EShLangFragment].pop(0);
 
-    builtInPoolAllocator->popAll();
     delete builtInPoolAllocator;
+    SetThreadPoolAllocator(savedGPA);
 
-    SetGlobalPoolAllocatorPtr(savedGPA);
+    glslang::ReleaseGlobalLock();
 }
 
 bool DeduceProfile(TInfoSink& infoSink, int version, EProfile& profile)
@@ -261,7 +279,7 @@ int ShInitialize()
         return 0;
 
     if (! PerProcessGPA) { 
-        PerProcessGPA = new TPoolAllocator(true);
+        PerProcessGPA = new TPoolAllocator();
     }
     
     glslang::TScanContext::fillInKeywordMap();
@@ -333,6 +351,7 @@ int __fastcall ShFinalize()
         PerProcessGPA->popAll();
         delete PerProcessGPA;
     }
+
     return 1;
 }
 
@@ -361,6 +380,7 @@ int ShCompile(
 
     if (handle == 0)
         return 0;
+
     TShHandleBase* base = reinterpret_cast<TShHandleBase*>(handle);
     TCompiler* compiler = base->getAsCompiler();
     if (compiler == 0)
@@ -372,7 +392,7 @@ int ShCompile(
     if (numStrings == 0)
         return 1;
 
-    GlobalPoolAllocator.push();
+    GetThreadPoolAllocator().push();
     
     // move to length-based strings, rather than null-terminated strings
     int* lengths = new int[numStrings];
@@ -395,7 +415,6 @@ int ShCompile(
     bool goodProfile = DeduceProfile(compiler->infoSink, version, profile);
 
     TIntermediate intermediate(compiler->infoSink, version, profile);
-    
     SetupBuiltinSymbolTable(version, profile);
     TSymbolTable symbolTable(*SharedSymbolTables[MapVersionToIndex(version)]
                                                 [profile]
@@ -480,7 +499,7 @@ int ShCompile(
     //
     // Throw away all the temporary memory used by the compilation process.
     //
-    GlobalPoolAllocator.pop();
+    GetThreadPoolAllocator().pop();
     delete [] lengths;
 
     return success ? 1 : 0;
@@ -510,9 +529,9 @@ int ShLink(
         return 0;
 
     int returnValue;
-    GlobalPoolAllocator.push();
+    GetThreadPoolAllocator().push();
     returnValue = ShLinkExt(linkHandle, compHandles, numHandles);
-    GlobalPoolAllocator.pop();
+    GetThreadPoolAllocator().pop();
 
     if (returnValue)
         return 1;

glslang/MachineIndependent/SymbolTable.h

@@ -77,7 +77,7 @@ class TFunction;
 class TAnonMember;
 class TSymbol {
 public:
-    POOL_ALLOCATOR_NEW_DELETE(GlobalPoolAllocator)
+    POOL_ALLOCATOR_NEW_DELETE(GetThreadPoolAllocator())
     explicit TSymbol(const TString *n) :  name(n) { }
 	virtual TSymbol* clone(TStructureMap& remapper) = 0;
     virtual ~TSymbol() { }
@@ -245,7 +245,7 @@ protected:
 
 class TSymbolTableLevel {
 public:
-    POOL_ALLOCATOR_NEW_DELETE(GlobalPoolAllocator)
+    POOL_ALLOCATOR_NEW_DELETE(GetThreadPoolAllocator())
     TSymbolTableLevel() : defaultPrecision (0), anonId(0) { }
 	~TSymbolTableLevel();
 
@@ -365,24 +365,23 @@ protected:
 
 class TSymbolTable {
 public:
-    TSymbolTable() : uniqueId(0), noBuiltInRedeclarations(false)
+    TSymbolTable() : uniqueId(0), noBuiltInRedeclarations(false), adoptedLevels(1)  // TODO: memory: can we make adoptedLevels be 0 for symbol tables we don't keep?
     {
         //
-        // The symbol table cannot be used until push() is called, but
-        // the lack of an initial call to push() can be used to detect
-        // that the symbol table has not been preloaded with built-ins.
+        // This symbol table cannot be used until push() is called.
         //
     }
     explicit TSymbolTable(TSymbolTable& symTable)
     {
         table.push_back(symTable.table[0]);
+        adoptedLevels = 1;
         uniqueId = symTable.uniqueId;
         noBuiltInRedeclarations = symTable.noBuiltInRedeclarations;
     }
     ~TSymbolTable()
     {
-        // level 0 is always built-in symbols, so we never pop that out
-        while (table.size() > 1)
+        // don't deallocate levels passed in from elsewhere
+        while (table.size() > adoptedLevels)
             pop(0);
     }
 
@@ -463,6 +462,7 @@ protected:
     std::vector<TSymbolTableLevel*> table;
     int uniqueId;     // for unique identification in code generation
     bool noBuiltInRedeclarations;
+    unsigned int adoptedLevels;
 };
 
 #endif // _SYMBOL_TABLE_INCLUDED_

glslang/MachineIndependent/glslang.y

@@ -319,6 +319,7 @@ postfix_expression
             TType newType($1->getType());
             newType.dereference();
             $$->setType(newType);
+            // TODO: functionality: does this drop const qualification for const[const] ?
         }
     }
     | function_call {

glslang/MachineIndependent/localintermediate.h

@@ -51,7 +51,7 @@ struct TVectorFields {
 class TInfoSink;
 class TIntermediate {
 public:    
-    POOL_ALLOCATOR_NEW_DELETE(GlobalPoolAllocator)
+    POOL_ALLOCATOR_NEW_DELETE(GetThreadPoolAllocator())
 
     TIntermediate(TInfoSink& i, int v, EProfile p) : infoSink(i), version(v), profile(p) { }
     TIntermSymbol* addSymbol(int Id, const TString&, const TType&, TSourceLoc);

glslang/OSDependent/Linux/osinclude.h

@@ -71,4 +71,18 @@ inline void * OS_GetTLSValue(OS_TLSIndex nIndex)
 	return pthread_getspecific(nIndex); 
 }
 
+namespace glslang {
+    void InitGlobalLock();
+    void GetGlobalLock();
+    void ReleaseGlobalLock();
+
+    typedef unsigned int (*TThreadEntrypoint)(void*);
+    void* OS_CreateThread(TThreadEntrypoint);
+    void OS_WaitForAllThreads(void* threads, int numThreads);
+
+    void OS_Sleep(int milliseconds);
+
+    void OS_DumpMemoryCounters();
+};
+
 #endif // __OSINCLUDE_H

glslang/OSDependent/Linux/ossource.cpp

@@ -134,3 +134,26 @@ bool OS_FreeTLSIndex(OS_TLSIndex nIndex)
 	else
 		return false;
 }
+
+namespace glslang {
+    // TODO: if we need these on linux, flesh them out
+    void InitGlobalLock() { }
+    void GetGlobalLock() { }
+    void ReleaseGlobalLock() { }
+
+    void* OS_CreateThread(TThreadEntrypoint entry)
+    {
+    }
+
+    void OS_WaitForAllThreads(void* threads, int numThreads)
+    {
+    }
+
+    void OS_Sleep(int milliseconds)
+    {
+    }
+
+    void OS_DumpMemoryCounters()
+    {
+    }
+};

glslang/OSDependent/Windows/main.cpp

@@ -34,6 +34,11 @@
 
 #include "InitializeDll.h"
 
+#define STRICT
+#define VC_EXTRALEAN 1
+#include <windows.h>
+#include <assert.h>
+
 BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved)
 {
 	switch (fdwReason)

glslang/OSDependent/Windows/osinclude.h

@@ -44,26 +44,30 @@
 #error Trying to include a windows specific file in a non windows build.
 #endif
 
-#define STRICT
-#define VC_EXTRALEAN 1
-#include <windows.h>
-#include <assert.h>
-
-
 //
 // Thread Local Storage Operations
 //
-typedef DWORD OS_TLSIndex;
-#define OS_INVALID_TLS_INDEX (TLS_OUT_OF_INDEXES)
+typedef unsigned long OS_TLSIndex;
+#define OS_INVALID_TLS_INDEX ((unsigned long)0xFFFFFFFF)
 
 OS_TLSIndex OS_AllocTLSIndex();
 bool        OS_SetTLSValue(OS_TLSIndex nIndex, void *lpvValue);
 bool        OS_FreeTLSIndex(OS_TLSIndex nIndex);
 
-inline void* OS_GetTLSValue(OS_TLSIndex nIndex)
-{
-	assert(nIndex != OS_INVALID_TLS_INDEX);
-	return TlsGetValue(nIndex);
-}
+void* OS_GetTLSValue(OS_TLSIndex nIndex);
+
+namespace glslang {
+    void InitGlobalLock();
+    void GetGlobalLock();
+    void ReleaseGlobalLock();
+
+    typedef unsigned int (__stdcall *TThreadEntrypoint)(void*);
+    void* OS_CreateThread(TThreadEntrypoint);
+    void OS_WaitForAllThreads(void* threads, int numThreads);
+
+    void OS_Sleep(int milliseconds);
+
+    void OS_DumpMemoryCounters();
+};
 
 #endif // __OSINCLUDE_H

glslang/OSDependent/Windows/ossource.cpp

@@ -33,6 +33,15 @@
 //
 
 #include "osinclude.h"
+
+#define STRICT
+#define VC_EXTRALEAN 1
+#include <windows.h>
+#include <assert.h>
+#include <process.h>
+#include <psapi.h>
+#include <stdio.h>
+
 //
 // This file contains contains the Window-OS-specific functions
 //
@@ -41,7 +50,6 @@
 #error Trying to build a windows specific file in a non windows build.
 #endif
 
-
 //
 // Thread Local Storage Operations
 //
@@ -70,6 +78,11 @@ bool OS_SetTLSValue(OS_TLSIndex nIndex, void *lpvValue)
 		return false;
 }
 
+void* OS_GetTLSValue(OS_TLSIndex nIndex)
+{
+	assert(nIndex != OS_INVALID_TLS_INDEX);
+	return TlsGetValue(nIndex);
+}
 
 bool OS_FreeTLSIndex(OS_TLSIndex nIndex)
 {
@@ -83,3 +96,45 @@ bool OS_FreeTLSIndex(OS_TLSIndex nIndex)
 	else
 		return false;
 }
+
+namespace glslang {
+    HANDLE GlobalLock;
+
+    void InitGlobalLock()
+    {
+        GlobalLock = CreateMutex(0, false, 0);
+    }
+
+    void GetGlobalLock()
+    {
+        WaitForSingleObject(GlobalLock, INFINITE);
+    }
+
+    void ReleaseGlobalLock()
+    {
+        ReleaseMutex(GlobalLock);
+    }
+
+    void* OS_CreateThread(TThreadEntrypoint entry)
+    {
+        return (void*)_beginthreadex(0, 0, entry, 0, 0, 0);
+        //return CreateThread(0, 0, entry, 0, 0, 0);
+    }
+
+    void OS_WaitForAllThreads(void* threads, int numThreads)
+    {
+        WaitForMultipleObjects(numThreads, (HANDLE*)threads, true, INFINITE);
+    }
+
+    void OS_Sleep(int milliseconds)
+    {
+        Sleep(milliseconds);
+    }
+
+    void OS_DumpMemoryCounters()
+    {
+        PROCESS_MEMORY_COUNTERS counters;
+        GetProcessMemoryInfo(GetCurrentProcess(), &counters, sizeof(counters));
+        printf("Working set size: %d\n", counters.WorkingSetSize);
+    }
+};