Merge pull request #3999 from samcake/temp0

Adding Scribe in tools

libraries/model/src/model/Geometry.h

@@ -63,8 +63,7 @@ public:
     // Access vertex position value
     const Vec3& getPos3(Index index) const { return _vertexBuffer.get<Vec3>(index); }
 
-    enum Topology
-    {
+    enum Topology {
         POINTS = 0,
         LINES,
         LINE_STRIP,

libraries/shared/src/Transform.h

@@ -63,20 +63,20 @@ public:
     void setIdentity();
 
     const Vec3& getTranslation() const;
-    void setTranslation(const Vec3& translation);
-    void preTranslate(const Vec3& translation);
-    void postTranslate(const Vec3& translation);
+    void setTranslation(const Vec3& translation); // [new this] = [translation] * [this.rotation] * [this.scale]
+    void preTranslate(const Vec3& translation);   // [new this] = [translation] * [this]
+    void postTranslate(const Vec3& translation);  // [new this] = [this] * [translation] equivalent to glTranslate
 
     const Quat& getRotation() const;
-    void setRotation(const Quat& rotation);
-    void preRotate(const Quat& rotation);
-    void postRotate(const Quat& rotation);
+    void setRotation(const Quat& rotation); // [new this] = [this.translation] * [rotation] * [this.scale]
+    void preRotate(const Quat& rotation);   // [new this] = [rotation] * [this]
+    void postRotate(const Quat& rotation);  // [new this] = [this] * [rotation] equivalent to glRotate
 
     const Vec3& getScale() const;
     void setScale(float scale);
-    void setScale(const Vec3& scale);
-    void postScale(float scale);
-    void postScale(const Vec3& scale);
+    void setScale(const Vec3& scale);  // [new this] = [this.translation] * [this.rotation] * [scale]
+    void postScale(float scale);       // [new this] = [this] * [scale] equivalent to glScale
+    void postScale(const Vec3& scale); // [new this] = [this] * [scale] equivalent to glScale
 
     bool isIdentity() const { return (_flags & ~Flags(FLAG_CACHE_INVALID_BITSET)).none(); }
     bool isTranslating() const { return _flags[FLAG_TRANSLATION]; }

tools/CMakeLists.txt

@@ -2,3 +2,4 @@
 add_subdirectory(bitstream2json)
 add_subdirectory(json2bitstream)
 add_subdirectory(mtc)
+add_subdirectory(scribe)

tools/scribe/CMakeLists.txt

@@ -0,0 +1,5 @@
+set(TARGET_NAME scribe)
+setup_hifi_project()
+
+# call macro to include our dependency includes and bubble them up via a property on our target
+include_dependency_includes()

tools/scribe/src/TextTemplate.cpp

@@ -0,0 +1,989 @@
+//
+//  TextTemplate.cpp
+//  tools/shaderScribe/src
+//
+//  Created by Sam Gateau on 12/15/2014.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+#include "TextTemplate.h"
+
+#include <stdarg.h>
+#include <fstream>
+#include <sstream>
+#include <algorithm>
+
+typedef TextTemplate::Block::Pointer BlockPointer;
+typedef TextTemplate::Config::Pointer ConfigPointer;
+typedef TextTemplate::Pointer TextTemplatePointer;
+
+//-----------------------------------------------------------------------------
+TextTemplate::Config::Config() :
+    _includes(),
+    _funcs(),
+    _logStream(&std::cout),
+    _numErrors(0),
+    _includerCallback(TextTemplate::loadFile) {
+    _paths.push_back("");
+}
+
+const TextTemplatePointer TextTemplate::Config::addInclude(const ConfigPointer& config, const char* include) {
+    if (!config) {
+        return TextTemplatePointer();
+    }
+
+    TextTemplatePointer included = config->findInclude(include);
+    if (included) {
+        return included;
+    }
+
+    // INcluded doest exist yet so let's try to create it
+    String includeStream;
+    if (config->_includerCallback(config, include, includeStream)) {
+        // ok, then create a new Template on the include file with this as lib
+        included = TextTemplatePointer(new TextTemplate(include, config));
+
+        std::stringstream src(includeStream);
+
+        int nbErrors = included->parse(src);
+        if (nbErrors > 0) {
+            included->logError(included->_root, "File failed to parse, not included");
+            return TextTemplatePointer();
+        }
+        config->_includes.insert(Includes::value_type(include, included));
+        return included;
+    }
+
+    return TextTemplatePointer();
+}
+
+const TextTemplatePointer TextTemplate::Config::findInclude(const char* include) {
+    Includes::iterator includeIt = _includes.find(String(include));
+    if (includeIt != _includes.end()) {
+        return (*includeIt).second;
+    } else {
+        return TextTemplatePointer();
+    }
+}
+
+void TextTemplate::Config::addIncludePath(const char* path) {
+    _paths.push_back(String(path));
+}
+
+bool TextTemplate::loadFile(const ConfigPointer& config, const char* filename, String& source) {
+    String sourceFile(filename);
+    String fullfilename;
+    for (unsigned int i = 0; i < config->_paths.size(); i++) {
+        fullfilename = config->_paths[i] + sourceFile;
+        std::ifstream ifs;
+        ifs.open(fullfilename.c_str());
+        if (ifs.is_open()) {
+            std::string str((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>());
+            source = str;
+            ifs.close();
+            return (source.length() > 0);
+        }
+    }
+
+    return false;
+}
+
+TextTemplate::Funcs::Funcs() :
+    _funcs() {
+}
+
+TextTemplate::Funcs::~Funcs() {
+}
+
+
+const BlockPointer TextTemplate::Funcs::findFunc(const char* func) {
+    map::iterator it = _funcs.find(String(func));
+    if (it != _funcs.end()) {
+        return (*it).second;
+    } else {
+        return BlockPointer();
+    }
+}
+
+const BlockPointer TextTemplate::Funcs::addFunc(const char* func, const BlockPointer& funcBlock) {
+    BlockPointer included = findFunc(func);
+    if (! included) {
+        _funcs.insert(map::value_type(func, funcBlock));
+    }
+    return included;
+}
+
+void TextTemplate::Block::addNewBlock(const Pointer& parent, const Pointer& block) {
+    if (parent) {
+        parent->blocks.push_back(block);
+        block->parent = parent;
+    }
+}
+
+const BlockPointer& TextTemplate::Block::getCurrentBlock(const Pointer& block) {
+    if (block && block->command.isBlockEnd()) {
+        return block->parent;
+    } else {
+        return block;
+    }
+}
+
+void TextTemplate::logError(const Block::Pointer& block, const char* fmt, ...) {
+    va_list argp;
+    va_start(argp, fmt);
+
+    char buff[256];
+    sprintf(buff, fmt, argp);
+
+    _numErrors++;
+    log() << block->sourceName << " Error >>" << buff << std::endl;
+
+    int level = 1;
+    displayTree(std::cerr, level);
+
+}
+
+bool TextTemplate::grabUntilBeginTag(std::istream* str, std::string& grabbed, Tag::Type& tagType) {
+    std::stringstream dst;
+    while (!str->eof()) {
+        
+        std::string datatoken;
+        getline((*str), datatoken, Tag::BEGIN);
+        dst << datatoken;
+            
+        char next = str->peek();
+        if (next == Tag::VAR) {
+            tagType = Tag::VARIABLE;
+            grabbed = dst.str();
+            str->get(); // skip tag char
+            return true;
+        } else if (next == Tag::COM) {
+            tagType = Tag::COMMAND;
+            grabbed = dst.str();
+            str->get(); // skip tag char
+            return true;
+        } else if (next == Tag::REM) {
+            tagType = Tag::REMARK;
+            grabbed = dst.str();
+            str->get(); // skip tag char
+            return true;
+        } else {
+            if (!str->eof()) {
+                // false positive, just found the Tag::BEGIN without consequence
+                dst << Tag::BEGIN;
+                // keep searching
+            } else {
+                // end of the file finishing with no tag
+                tagType = Tag::INVALID;
+                grabbed = dst.str();
+                return true;
+            }
+        }
+    }
+
+    return false;
+}
+
+bool TextTemplate::grabUntilEndTag(std::istream* str, std::string& grabbed, Tag::Type& tagType) {
+    std::stringstream dst;
+
+    // preEnd char depends on tag type
+    char preEnd = Tag::COM;
+    if (tagType == Tag::VARIABLE) {
+        preEnd = Tag::VAR;
+    } else if (tagType == Tag::REMARK) {
+        preEnd = Tag::REM;
+    }
+
+    while (!str->eof()) {
+        // looking for the end of the tag means find the next preEnd
+        std::string dataToken;
+        getline((*str), dataToken, preEnd);
+        char end = str->peek();
+
+        dst << dataToken;
+
+        // and if the next char is Tag::END then that's it
+        if (end == Tag::END) {
+            grabbed = dst.str();
+            str->get(); // eat the Tag::END
+            return true;
+        } else {
+            // false positive, keep on searching
+            dst << preEnd;
+        }
+    }
+    grabbed = dst.str();
+    return false;
+}
+
+bool TextTemplate::stepForward(std::istream* str, std::string& grabbed, std::string& tag, Tag::Type& tagType,
+    Tag::Type& nextTagType) {
+    if (str->eof()) {
+        return false;
+    }
+            
+    if (!_steppingStarted) {
+        _steppingStarted = true;
+        return grabUntilBeginTag(str, grabbed, nextTagType);
+    }
+
+    // Read from the last opening Tag::BEGIN captured to the next Tag::END
+    if (grabUntilEndTag(str, tag, tagType)) {
+        // skip trailing space and new lines only after Command or Remark tag block
+        if ((tagType == Tag::COMMAND) || (tagType == Tag::REMARK)) {
+            while (!str->eof()) {
+                char c = str->peek();
+                if ((c == ' ') || (c == '\t') || (c == '\n')) {
+                    str->get();
+                } else {
+                    break;
+                }
+            }
+        }
+
+        grabUntilBeginTag(str, grabbed, nextTagType);
+    }
+
+    return true; //hasElement;
+}
+
+const BlockPointer TextTemplate::processStep(const BlockPointer& block, std::string& grabbed, std::string& tag,
+    Tag::Type& tagType) {
+    switch (tagType) {
+        case Tag::INVALID:
+            block->ostr << grabbed;
+            return block;
+            break;
+        case Tag::VARIABLE:
+            return processStepVar(block, grabbed, tag);
+            break;
+        case Tag::COMMAND:
+            return processStepCommand(block, grabbed, tag);
+            break;
+        case Tag::REMARK:
+            return processStepRemark(block, grabbed, tag);
+            break;
+    }
+
+    logError(block, "Invalid tag");
+    return block;
+}
+
+bool TextTemplate::grabFirstToken(String& src, String& token, String& reminder) {
+    bool goOn = true;
+    std::string::size_type i = 0;
+    while (goOn && (i < src.length())) {
+        char c = src[i];
+        if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || (c == '_') || (c == '.') || (c == '[') || (c == ']')) {
+            token += c;
+        } else {
+            if (!token.empty()) {
+                reminder = src.substr(i);
+                return true;
+            }
+        }
+        i++;
+    }
+
+    return (!token.empty());
+}
+
+bool TextTemplate::convertExpressionToArguments(String& src, std::vector< String >& arguments) {
+    std::stringstream str(src);
+    String token;
+
+    while (!str.eof()) {
+        str >> token;
+        if (!str.fail()) {
+            arguments.push_back(token);
+        }
+    }
+
+    return true;
+}
+
+bool TextTemplate::convertExpressionToDefArguments(String& src, std::vector< String >& arguments) {
+    if (src.empty()) {
+        return false;
+    }
+
+    std::stringstream argstr(src);
+    std::stringstream dest;
+    bool inVar = false;
+    while (!argstr.eof()) {
+        // parse the value of a var, try to find a VAR, so look for the pattern BEGIN,VAR ... VAR,END
+        String token;
+        char tag;
+        if (!inVar) {
+            getline(argstr, token, Tag::BEGIN);
+            dest << token;
+            tag = argstr.peek();
+        } else {
+            getline(argstr, token, Tag::END);
+            dest << token;
+            tag = token.back();
+        }
+
+        if (tag == Tag::VAR) {
+            if (!inVar) {
+                // real var coming:
+                arguments.push_back(dest.str());
+                inVar = true;
+            } else {
+                // real var over
+                arguments.push_back(dest.str());
+                inVar = false;
+            }
+        } else {
+            if (argstr.eof()) {
+                arguments.push_back(dest.str());
+            } else {
+                // put back the tag char stolen
+                dest << (!inVar ? Tag::BEGIN : Tag::END);
+            }
+        }
+    }
+
+    return true;
+}
+
+bool TextTemplate::convertExpressionToFuncArguments(String& src, std::vector< String >& arguments) {
+    if (src.empty()) {
+        return false;
+    }
+    std::stringstream streamSrc(src);
+    String params;
+    getline(streamSrc, params, '(');
+    getline(streamSrc, params, ')');
+    if (params.empty()) {
+        return false;
+    }
+
+    std::stringstream str(params);
+    String token;
+    int nbTokens = 0;
+    while (!str.eof()) {
+        char c = str.peek();
+        if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || (c == '_') || (c == '.') || (c == Tag::VAR)  || (c == '[') || (c == ']')) {
+            token += c;
+        } else if (c == ',') {
+            if (!token.empty()) {
+                arguments.push_back(token);
+                nbTokens++;
+            }
+            token.clear();
+        }
+
+        str.get();
+    }
+
+    if (token.size()) {
+        arguments.push_back(token);
+        nbTokens++;
+    }
+
+    return (nbTokens > 0);
+}
+
+const BlockPointer TextTemplate::processStepVar(const BlockPointer& block, String& grabbed, String& tag) {
+    // then look at the define
+    String var = tag;
+    String varName;
+    String val;
+
+    if (grabFirstToken(var, varName, val)) {
+        if (!varName.empty()) {
+            BlockPointer parent = Block::getCurrentBlock(block);
+
+            // Add a new BLock 
+            BlockPointer newBlock = BlockPointer(new Block(_root->sourceName));
+            (newBlock->ostr) << grabbed;
+
+            newBlock->command.type = Command::VAR;
+
+            newBlock->command.arguments.push_back(varName);
+
+            if (!val.empty()) {
+                convertExpressionToFuncArguments(val, newBlock->command.arguments);
+            }
+
+            Block::addNewBlock(parent, newBlock);
+
+            // dive in the new block
+            return newBlock;
+        }
+    }
+
+    return block;
+}
+
+const BlockPointer TextTemplate::processStepCommand(const BlockPointer& block, String& grabbed, String& tag) {
+    // Grab the command name
+    String command = tag;
+    String commandName;
+
+    if (grabFirstToken(command, commandName, command)) {
+        if (commandName.compare("def") == 0) {
+            return processStepDef(block, grabbed, command);
+        } else if (commandName.compare("if") == 0) {
+            return processStepCommandIf(block, grabbed, command);
+        } else if (commandName.compare("endif") == 0) {
+            return processStepCommandEndIf(block, grabbed, command);
+        } else if (commandName.compare("else") == 0) {
+            return processStepCommandElse(block, grabbed, command);
+        } else if (commandName.compare("elif") == 0) {
+            return processStepCommandElif(block, grabbed, command);
+        } else if (commandName.compare("include") == 0) {
+            return processStepInclude(block, grabbed, command);
+        } else if (commandName.compare("func") == 0) {
+            return processStepFunc(block, grabbed, command);
+        } else if (commandName.compare("endfunc") == 0) {
+            return processStepEndFunc(block, grabbed, command);
+        }
+    }
+
+    return block;
+}
+
+const BlockPointer TextTemplate::processStepDef(const BlockPointer& block, String& grabbed, String& def) {
+    // then look at the define
+    String varName;
+    String val;
+
+    if (!grabFirstToken(def, varName, val)) {
+        logError(block, "Invalid Var name in a <def> tag");
+        return block;
+    }
+
+    BlockPointer parent = Block::getCurrentBlock(block);
+
+    // Add a new BLock 
+    BlockPointer newBlock = BlockPointer(new Block(_root->sourceName));
+    (newBlock->ostr) << grabbed;
+
+    newBlock->command.type = Command::DEF;
+    newBlock->command.arguments.push_back(varName);
+    if (!val.empty()) {
+        // loose first character which should be a white space
+        val = val.substr(val.find_first_not_of(' '));
+        convertExpressionToDefArguments(val, newBlock->command.arguments);
+    }
+
+    Block::addNewBlock(parent, newBlock);
+
+    // dive in the new block
+    return newBlock;
+}
+
+
+const BlockPointer TextTemplate::processStepCommandIf(const BlockPointer& block, String& grabbed, String& expression) {
+    BlockPointer parent = Block::getCurrentBlock(block);
+
+    // Add a new BLock depth
+    BlockPointer newIfBlock = BlockPointer(new Block(_root->sourceName));
+    newIfBlock->command.type = Command::IFBLOCK;
+
+    Block::addNewBlock(parent, newIfBlock);
+
+    BlockPointer newBlock = BlockPointer(new Block(_root->sourceName));
+    (newBlock->ostr) << grabbed;
+
+    newBlock->command.type = Command::IF;
+    convertExpressionToArguments(expression, newBlock->command.arguments);
+
+    Block::addNewBlock(newIfBlock, newBlock);
+
+    // dive in the new If branch
+    return newBlock;
+}
+
+const BlockPointer TextTemplate::processStepCommandEndIf(const BlockPointer& block, String& grabbed, String& expression) {
+    BlockPointer parent = Block::getCurrentBlock(block);
+
+    // are we in a if block ?
+    if ((parent->command.type == Command::IF) 
+        ||  (parent->command.type == Command::ELIF)
+        ||  (parent->command.type == Command::ELSE)) {
+        BlockPointer newBlock = BlockPointer(new Block(_root->sourceName));
+        (newBlock->ostr) << grabbed;
+
+        newBlock->command.type = Command::ENDIF;
+        newBlock->command.arguments.push_back(expression);
+
+        Block::addNewBlock(parent->parent->parent, newBlock);
+
+        return newBlock;
+    } else {
+        logError(block, "Invalid <endif> block, not in a <if> block");
+        return block;
+    }
+
+    return block;
+}
+
+const BlockPointer TextTemplate::processStepCommandElse(const BlockPointer& block, String& grabbed, String& expression) {
+    BlockPointer parent = Block::getCurrentBlock(block);
+
+    // are we in a if block ?
+    if ((parent->command.type == Command::IF) 
+        ||  (parent->command.type == Command::ELIF)) {
+        // All good go back to the IfBlock
+        parent = parent->parent;
+
+        // Add a new BLock depth
+        BlockPointer newBlock = BlockPointer(new Block(_root->sourceName));
+        newBlock->ostr << grabbed;
+        newBlock->command.type = Command::ELSE;
+        newBlock->command.arguments.push_back(expression);
+
+        Block::addNewBlock(parent, newBlock);
+
+        // dive in the new block
+        return newBlock;
+
+    } else if ((block)->command.type == Command::ELSE) {
+        logError(block, "Invalid <elif> block, in a <if> block but after a <else> block");
+        return block;
+    } else {
+        logError(block, "Invalid <elif> block, not in a <if> block");
+        return block;
+    }
+}
+
+const BlockPointer TextTemplate::processStepCommandElif(const BlockPointer& block, String& grabbed, String& expression) {
+    BlockPointer parent = Block::getCurrentBlock(block);
+
+    // are we in a if block ?
+    if ((parent->command.type == Command::IF)
+        ||  (parent->command.type == Command::ELIF)) {
+        // All good go back to the IfBlock
+        parent = parent->parent;
+
+        // Add a new BLock depth
+        BlockPointer newBlock = BlockPointer(new Block(_root->sourceName));
+        (newBlock->ostr) << grabbed;
+
+        newBlock->command.type = Command::ELIF;
+        convertExpressionToArguments(expression, newBlock->command.arguments);
+
+        Block::addNewBlock(parent, newBlock);
+
+        // dive in the new block
+        return newBlock;
+
+    } else if (parent->command.type == Command::ELSE) {
+        logError(block, "Invalid <elif> block, in a <if> block but after a <else> block");
+        return block;
+    } else {
+        logError(block, "Invalid <elif> block, not in a <if> block");
+        return block;
+    }
+}
+
+const BlockPointer TextTemplate::processStepRemark(const BlockPointer& block, String& grabbed, String& tag) {
+    // nothing to do :)
+
+    // no need to think, let's just add the grabbed text
+    (block->ostr) << grabbed;
+
+    return block;
+}
+
+const BlockPointer TextTemplate::processStepInclude(const BlockPointer& block, String& grabbed, String& include) {
+    BlockPointer parent = Block::getCurrentBlock(block);
+
+    // Add a new BLock 
+    BlockPointer newBlock = BlockPointer(new Block(_root->sourceName));
+    (newBlock->ostr) << grabbed;
+
+    newBlock->command.type = Command::INCLUDE;
+    convertExpressionToArguments(include, newBlock->command.arguments);
+
+    Block::addNewBlock(parent, newBlock);
+
+    TextTemplatePointer inc = Config::addInclude(_config, newBlock->command.arguments.front().c_str());
+    if (!inc) {
+        logError(newBlock, "Failed to include %s", newBlock->command.arguments.front().c_str());
+    }
+
+    // dive in the new block
+    return newBlock;
+}
+
+const BlockPointer TextTemplate::processStepFunc(const BlockPointer& block, String& grabbed, String& func) {
+    // then look at the define
+    String varName;
+    String var;
+
+    if (!grabFirstToken(func, varName, var)) {
+        logError(block, "Invalid func name <%s> in a <func> block", func.c_str());
+        return block;
+    }
+
+    // DOes the func already exists ?
+    if (_config->_funcs.findFunc(varName.c_str())) {
+        logError(block, "Declaring a new func named <%s> already exists", func.c_str());
+        return block;
+    }
+
+    BlockPointer parent = Block::getCurrentBlock(block);
+
+    // Add a new BLock 
+    BlockPointer newBlock = BlockPointer(new Block(_root->sourceName));
+    (newBlock->ostr) << grabbed;
+
+    newBlock->command.type = Command::FUNC;
+    newBlock->command.arguments.push_back(varName);
+    convertExpressionToFuncArguments(var, newBlock->command.arguments);
+
+    _config->_funcs.addFunc(varName.c_str(), newBlock);
+
+    Block::addNewBlock(parent, newBlock);
+
+    // dive in the new block
+    return newBlock;
+}
+
+const BlockPointer TextTemplate::processStepEndFunc(const BlockPointer& block, String& grabbed, String& tag) {
+    BlockPointer parent = Block::getCurrentBlock(block);
+
+    // are we in a func block ?
+    if (parent->command.type == Command::FUNC) {
+        // Make sure the FUnc has been declared properly
+        BlockPointer funcBlock = _config->_funcs.findFunc(parent->command.arguments.front().c_str());
+        if (funcBlock != parent) {
+            logError(block, "Mismatching <func> blocks");
+            return BlockPointer();
+        }
+
+        // Everything is cool , so let's unplugg the FUnc block from this tree and just put the EndFunc block
+
+        BlockPointer newBlock = BlockPointer(new Block(_root->sourceName));
+        (newBlock->ostr) << grabbed;
+
+        newBlock->command.type = Command::ENDFUNC;
+        convertExpressionToArguments(tag, newBlock->command.arguments);
+
+        newBlock->parent = parent->parent;
+
+        parent->parent->blocks.back() = newBlock;
+
+        parent->parent = 0;
+
+        // dive in the new block
+        return newBlock;
+    } else {
+        logError(block, "Invalid <endfunc> block, not in a <func> block");
+        return BlockPointer();
+    }
+
+    return block;
+}
+
+TextTemplate::TextTemplate(const String& name, const ConfigPointer& externalConfig) :
+    _config(externalConfig),
+    _root(new Block(name)),
+    _numErrors(0),
+    _steppingStarted(false) {
+}
+
+TextTemplate::~TextTemplate() {
+}
+
+int TextTemplate::scribe(std::ostream& dst, std::istream& src, Vars& vars) {
+    int nbErrors = parse(src);
+
+    if (nbErrors == 0) {
+        return generate(dst, vars);
+    }
+    return nbErrors;
+}
+
+int TextTemplate::generateTree(std::ostream& dst, const BlockPointer& block, Vars& vars) {
+    BlockPointer newCurrentBlock;
+    int numPasses = evalBlockGeneration(dst, block, vars, newCurrentBlock);
+    for (int passNum= 0; passNum < numPasses; passNum++) {
+        dst << newCurrentBlock->ostr.str();
+
+        for (auto child : newCurrentBlock->blocks) {
+            generateTree(dst, child, vars);
+        }
+    }
+
+    return _numErrors;
+}
+
+int TextTemplate::evalBlockGeneration(std::ostream& dst, const BlockPointer& block, Vars& vars, BlockPointer& branch) {
+    switch (block->command.type) {
+        case Command::BLOCK: {
+            branch = block;
+            return 1;
+        }
+        break;
+        case Command::VAR: {
+            Vars::iterator it = vars.find(block->command.arguments.front());
+            if (it != vars.end()) {
+                dst << (*it).second;
+            } else {
+                BlockPointer funcBlock = _config->_funcs.findFunc(block->command.arguments.front().c_str());
+                if (funcBlock) {
+                    // before diving in the func tree, let's modify the vars with the local defs:
+                    int nbParams = std::min(block->command.arguments.size(), funcBlock->command.arguments.size());
+                    std::vector< String > paramCache;
+                    paramCache.push_back("");
+                    String val;
+                    for (int i = 1; i < nbParams; i++) {
+                        val = block->command.arguments[i];
+                        if ((val[0] == Tag::VAR) && (val[val.length()-1] == Tag::VAR)) {
+                            val = val.substr(1, val.length()-2);
+                            Vars::iterator it = vars.find(val);
+                            if (it != vars.end()) {
+                                val = (*it).second;
+                            }
+                        }
+
+                        Vars::iterator it = vars.find(funcBlock->command.arguments[i]);
+                        if (it != vars.end()) {
+                            paramCache.push_back((*it).second);
+                            (*it).second = val;
+                        } else {
+                            vars.insert(Vars::value_type(funcBlock->command.arguments[i], val));
+                            paramCache.push_back("");
+                        }
+                    }
+
+                    generateTree(dst, funcBlock, vars);
+
+                    for (int i = 1; i < nbParams; i++) {
+                        vars[ funcBlock->command.arguments[i] ] = paramCache[i];
+                    }
+                }
+            }
+            branch = block;
+            return 1;
+        }
+        break;
+        case Command::IFBLOCK: {
+            // ok, go through the branches and pick the first one that goes
+            for (auto child: block->blocks) {
+                int numPasses = evalBlockGeneration(dst, child, vars, branch);
+                if (numPasses > 0) {
+                    return numPasses;
+                }
+            }
+        }
+        break;
+        case Command::IF:
+        case Command::ELIF: {
+            if (!block->command.arguments.empty()) {
+                // Just one argument means check for the var beeing defined
+                if (block->command.arguments.size() == 1) {
+                    Vars::iterator it = vars.find(block->command.arguments.front());
+                    if (it != vars.end()) {
+                        branch = block;
+                        return 1;
+                    }
+                } else if (block->command.arguments.size() == 2) {
+                    if (block->command.arguments[0].compare("not") == 0) {
+                        Vars::iterator it = vars.find(block->command.arguments[1]);
+                        if (it == vars.end()) {
+                            branch = block;
+                            return 1;
+                        }
+                    }
+                } else if (block->command.arguments.size() == 3) {
+                    if (block->command.arguments[1].compare("and") == 0) {
+                        Vars::iterator itL = vars.find(block->command.arguments[0]);
+                        Vars::iterator itR = vars.find(block->command.arguments[2]);
+                        if ((itL != vars.end()) && (itR != vars.end())) {
+                            branch = block;
+                            return 1;
+                        }
+                    } else if (block->command.arguments[1].compare("or") == 0) {
+                        Vars::iterator itL = vars.find(block->command.arguments[0]);
+                        Vars::iterator itR = vars.find(block->command.arguments[2]);
+                        if ((itL != vars.end()) || (itR != vars.end())) {
+                            branch = block;
+                            return 1;
+                        }
+                    } else if (block->command.arguments[1].compare("==") == 0) {
+                        Vars::iterator itL = vars.find(block->command.arguments[0]);
+                        if (itL != vars.end()) {
+                            if ((*itL).second.compare(block->command.arguments[2]) == 0) {
+                                branch = block;
+                                return 1;
+                            }
+                        }
+                    }
+                }
+
+            }
+            return 0;
+        }
+        break;
+        case Command::ELSE: {
+            branch = block;
+            return 1;
+        }
+        break;
+        case Command::ENDIF: {
+            branch = block;
+            return 1;
+        }
+        break;
+        case Command::DEF: {
+            if (block->command.arguments.size()) {
+                // THe actual value of the var defined sneeds to be evaluated:
+                String val;
+                for (int t = 1; t < block->command.arguments.size(); t++) {
+                    // detect if a param is a var
+                    int len = block->command.arguments[t].length();
+                    if ((block->command.arguments[t][0] == Tag::VAR)
+                        && (block->command.arguments[t][len - 1] == Tag::VAR)) {
+                        String var = block->command.arguments[t].substr(1, len - 2);
+                        Vars::iterator it = vars.find(var);
+                        if (it != vars.end()) {
+                            val += (*it).second;
+                        }
+                    } else {
+                        val += block->command.arguments[t];
+                    }
+                }
+
+                Vars::iterator it = vars.find(block->command.arguments.front());
+                if (it == vars.end()) {
+                    vars.insert(Vars::value_type(block->command.arguments.front(), val));
+                } else {
+                    (*it).second = val;
+                }
+
+                branch = block;
+                return 1;
+            } else {
+                branch = block;
+                return 0;
+            }
+        }
+        break;
+
+        case Command::INCLUDE: {
+            TextTemplatePointer include = _config->findInclude(block->command.arguments.front().c_str());
+            if (include && !include->_root->blocks.empty()) {
+                if (&include->_root) {
+                    generateTree(dst, include->_root, vars);
+                }
+            }
+
+            branch = block;
+            return 1;
+        }
+        break;
+
+        case Command::FUNC: {
+            branch = block;
+            return 1;
+        }
+        break;
+
+        case Command::ENDFUNC: {
+            branch = block;
+            return 1;
+        }
+        break;
+
+        default: {
+        }
+    }
+
+    return 0;
+}
+
+
+int TextTemplate::parse(std::istream& src) {
+    _root->command.type = Command::BLOCK;
+    BlockPointer currentBlock = _root;
+    _numErrors = 0;
+
+    // First Parse the input file
+    int nbLoops = 0;
+    bool goOn = true;
+    Tag::Type tagType = Tag::INVALID;
+    Tag::Type nextTagType = Tag::INVALID;
+    while (goOn) {
+        std::string data;
+        std::string tag;
+        if (stepForward(&src, data, tag, tagType, nextTagType)) {
+            currentBlock = processStep(currentBlock, data, tag, tagType);
+        } else {
+            goOn = false;
+        }
+
+        tagType = nextTagType;
+        nbLoops++;
+    }
+
+    return _numErrors;
+}
+
+int TextTemplate::generate(std::ostream& dst, Vars& vars) {
+    return generateTree(dst, _root, vars);
+}
+
+void TextTemplate::displayTree(std::ostream& dst, int& level) const {
+    Block::displayTree(_root, dst, level);
+}
+
+void TextTemplate::Block::displayTree(const BlockPointer& block, std::ostream& dst, int& level) {
+    String tab(level * 2, ' ');
+
+    const String BLOCK_TYPE_NAMES[] = {
+        "VAR",
+        "BLOCK",
+        "FUNC",
+        "ENDFUNC",
+        "IFBLOCK",
+        "IF",
+        "ELIF",
+        "ELSE",
+        "ENDIF",
+        "FOR",
+        "ENDFOR",
+        "INCLUDE",
+        "DEF"
+    };
+
+    dst << tab << "{ " << BLOCK_TYPE_NAMES[block->command.type] << ":";
+    if (!block->command.arguments.empty()) {
+        for (auto arg: block->command.arguments) {
+            dst << " " << arg;
+        }
+    }
+    dst << std::endl;
+
+    level++;
+    for (auto sub: block->blocks) {
+        displayTree(sub, dst, level);
+    }
+    level--;
+
+    dst << tab << "}" << std::endl;
+}
+
+void TextTemplate::Config::displayTree(std::ostream& dst, int& level) const {
+    String tab(level * 2, ' ');
+
+    level++;
+    dst << tab << "Includes:" << std::endl;
+    for (auto inc: _includes) {
+        dst << tab << tab << inc.first << std::endl;
+        inc.second->displayTree(dst, level);
+    }
+    dst << tab << "Funcs:" << std::endl;
+    for (auto func: _funcs._funcs) {
+        dst << tab << tab << func.first << std::endl;
+        TextTemplate::Block::displayTree( func.second, dst, level);
+    }
+    level--;
+}

tools/scribe/src/TextTemplate.h

@@ -0,0 +1,204 @@
+
+//
+//  TextTemplate.cpp
+//  tools/shaderScribe/src
+//
+//  Created by Sam Gateau on 12/15/2014.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+#ifndef hifi_TEXT_TEMPLATE_H
+#define hifi_TEXT_TEMPLATE_H
+
+#include <list>
+#include <vector>
+#include <map>
+#include <string>
+#include <iostream>
+#include <sstream>
+#include <memory>
+
+class TextTemplate {
+public:
+    typedef std::shared_ptr< TextTemplate > Pointer;
+    typedef std::string String;
+    typedef std::vector< String > StringVector;
+    typedef std::map< String, String > Vars;
+    typedef std::map< String, TextTemplate::Pointer > Includes;
+
+    class Tag {
+    public:
+        enum Type {
+            VARIABLE = 0,
+            COMMAND,
+            REMARK,
+            INVALID = -1,
+        };
+
+        static const char BEGIN = '<';
+        static const char END = '>';
+
+        static const char VAR = '$';
+        static const char COM = '@';
+        static const char REM = '!';
+    };
+
+    class Command {
+    public:
+        typedef std::vector< Command > vector;
+
+        enum Type {
+            VAR = 0,
+            BLOCK,
+            FUNC,
+            ENDFUNC,
+            IFBLOCK,
+            IF,
+            ELIF,
+            ELSE,
+            ENDIF,
+            FOR,
+            ENDFOR,
+            INCLUDE,
+            DEF,
+        };
+
+        Type type;
+        std::vector< String > arguments;
+
+        bool isBlockEnd() {
+            switch (type) {
+                case ENDFUNC:
+                case ENDIF:
+                case ENDFOR:
+                case INCLUDE:
+                case DEF:
+                case VAR:
+                    return true;
+                default:
+                    return false;
+            }
+        }
+    };
+
+    class Block {
+    public:
+        typedef std::shared_ptr<Block> Pointer;
+        typedef std::vector< Block::Pointer > Vector;
+
+        Block::Pointer parent;
+        Command command;
+        Vector blocks;
+        std::stringstream ostr;
+
+        String sourceName;
+
+        Block(const String& sourceFilename) :
+            sourceName(sourceFilename) {}
+
+        static void addNewBlock(const Block::Pointer& parent, const Block::Pointer& block);
+        static const Block::Pointer& getCurrentBlock(const Block::Pointer& block);
+
+        static void displayTree(const Block::Pointer& block, std::ostream& dst, int& level);
+    };
+
+    class Funcs {
+    public:
+        typedef std::map< String, Block::Pointer > map;
+
+        Funcs();
+        ~Funcs();
+
+        const Block::Pointer findFunc(const char* func);
+        const Block::Pointer addFunc(const char* func, const Block::Pointer& root);
+
+        map _funcs;
+    protected:
+    };
+
+    class Config {
+    public:
+        typedef std::shared_ptr< Config > Pointer;
+        typedef bool (*IncluderCallback) (const Config::Pointer& config, const char* filename, String& source);
+
+        Includes        _includes;
+        Funcs           _funcs;
+        std::ostream*   _logStream;
+        int             _numErrors;
+        IncluderCallback _includerCallback;
+        StringVector    _paths;
+
+        Config();
+
+        static const TextTemplate::Pointer addInclude(const Config::Pointer& config, const char* include);
+        const TextTemplate::Pointer findInclude(const char* include);
+
+        void addIncludePath(const char* path);
+
+        void displayTree(std::ostream& dst, int& level) const;
+    };
+
+    static bool loadFile(const Config::Pointer& config, const char* filename, String& source);
+
+    TextTemplate(const String& name, const Config::Pointer& config = Config::Pointer(new Config()));
+    ~TextTemplate();
+
+    // Scibe does all the job of parsing an inout template stream and then gneerating theresulting stream using the vars
+    int scribe(std::ostream& dst, std::istream& src, Vars& vars);
+
+    int parse(std::istream& src);
+    int generate(std::ostream& dst, Vars& vars);
+
+    const Config::Pointer config() { return _config; }
+
+    void displayTree(std::ostream& dst, int& level) const;
+
+protected:
+    Config::Pointer _config;
+    Block::Pointer _root;
+    int _numErrors;
+    bool _steppingStarted;
+
+    bool grabUntilBeginTag(std::istream* str, String& grabbed, Tag::Type& tagType);
+    bool grabUntilEndTag(std::istream* str, String& grabbed, Tag::Type& tagType);
+
+    bool stepForward(std::istream* str, String& grabbed, String& tag, Tag::Type& tagType, Tag::Type& nextTagType);
+
+    bool grabFirstToken(String& src, String& token, String& reminder);
+    bool convertExpressionToArguments(String& src, std::vector< String >& arguments);
+    bool convertExpressionToDefArguments(String& src, std::vector< String >& arguments);
+    bool convertExpressionToFuncArguments(String& src, std::vector< String >& arguments);
+
+    // Filter between var, command or comments
+    const Block::Pointer processStep(const Block::Pointer& block, String& grabbed, String& tag, Tag::Type& tagType);
+    const Block::Pointer processStepVar(const Block::Pointer& block, String& grabbed, String& tag);
+    const Block::Pointer processStepCommand(const Block::Pointer& block, String& grabbed, String& tag);
+    const Block::Pointer processStepRemark(const Block::Pointer& block, String& grabbed, String& tag);
+
+    // Define command
+    const Block::Pointer processStepDef(const Block::Pointer& block, String& grabbed, String& tag);
+
+    // If commands
+    const Block::Pointer processStepCommandIf(const Block::Pointer& block, String& grabbed, String& expression);
+    const Block::Pointer processStepCommandEndIf(const Block::Pointer& block, String& grabbed, String& expression);
+    const Block::Pointer processStepCommandElif(const Block::Pointer& block, String& grabbed, String& expression);
+    const Block::Pointer processStepCommandElse(const Block::Pointer& block, String& grabbed, String& expression);
+
+    // Include command
+    const Block::Pointer processStepInclude(const Block::Pointer& block, String& grabbed, String& tag);
+
+    // Function command
+    const Block::Pointer processStepFunc(const Block::Pointer& block, String& grabbed, String& tag);
+    const Block::Pointer processStepEndFunc(const Block::Pointer& block, String& grabbed, String& tag);
+
+    // Generation
+    int generateTree(std::ostream& dst, const Block::Pointer& block, Vars& vars);
+    int evalBlockGeneration(std::ostream& dst, const Block::Pointer& block, Vars& vars, Block::Pointer& branch);
+
+    // Errors
+    std::ostream& log() { return (* _config->_logStream); }
+    void logError(const Block::Pointer& block, const char* error, ...);
+};
+
+#endif

tools/scribe/src/main.cpp

@@ -0,0 +1,225 @@
+//
+//  main.cpp
+//  tools/shaderScribe/src
+//
+//  Created by Sam Gateau on 12/15/2014.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+
+
+#include "TextTemplate.h"
+
+#include <fstream>
+#include <ctime>
+#include <chrono>
+
+using namespace std;
+
+int main (int argc, char** argv) {
+    // process the command line arguments
+    std::vector< std::string > inputs;
+
+    std::string srcFilename;
+    std::string destFilename;
+    std::string targetName;
+    TextTemplate::Vars vars;
+    
+    std::string lastVarName;
+    bool listVars = false;
+    bool showParseTree = false;
+    bool makeCPlusPlus = false;
+
+    TextTemplate::Config::Pointer config(new TextTemplate::Config());
+
+    enum Mode {
+        READY = 0,
+        GRAB_OUTPUT,
+        GRAB_VAR_NAME,
+        GRAB_VAR_VALUE,
+        GRAB_INCLUDE_PATH,
+        GRAB_TARGET_NAME,
+        EXIT,
+    } mode = READY;
+
+    for (int ii = 1; (mode != EXIT) && (ii < argc); ii++) {
+        inputs.push_back(argv[ii]);
+
+        switch (mode) {
+            case READY: {
+                if (inputs.back() == "-o") {
+                    mode = GRAB_OUTPUT;
+                } else if (inputs.back() == "-tn") {
+                    mode = GRAB_TARGET_NAME;
+                } else if (inputs.back() == "-D") {
+                    mode = GRAB_VAR_NAME;
+                } else if (inputs.back() == "-I") {
+                    mode = GRAB_INCLUDE_PATH;
+                } else if (inputs.back() == "-listVars") {
+                    listVars = true;
+                    mode = READY;
+                } else if (inputs.back() == "-showParseTree") {
+                    showParseTree = true;
+                    mode = READY;
+                } else if (inputs.back() == "-c++") {
+                    makeCPlusPlus = true;
+                    mode = READY;
+                } else {
+                    // just grabbed the source filename, stop parameter parsing
+                    srcFilename = inputs.back();
+                    mode = EXIT;
+                }
+            }
+            break;
+
+            case GRAB_OUTPUT: {
+                destFilename = inputs.back();
+                mode = READY;
+            }
+            break;
+
+            case GRAB_TARGET_NAME: {
+                targetName = inputs.back();
+                mode = READY;
+            }
+            break;
+
+            case GRAB_VAR_NAME: {
+                // grab first the name of the var
+                lastVarName = inputs.back();
+                mode = GRAB_VAR_VALUE;
+            }
+            break;
+            case GRAB_VAR_VALUE: {
+                // and then the value
+                vars.insert(TextTemplate::Vars::value_type(lastVarName, inputs.back()));
+
+                mode = READY;
+            }
+            break;
+
+            case GRAB_INCLUDE_PATH: {
+                config->addIncludePath(inputs.back().c_str());
+                mode = READY;
+            }
+            break;
+                
+            case EXIT: {
+                // THis shouldn't happen
+            }
+            break;
+        }
+    }
+
+    if (srcFilename.empty()) {
+        cerr << "Usage: shaderScribe [OPTION]... inputFilename" << endl;
+        cerr << "Where options include:" << endl;
+        cerr << "  -o filename: Send output to filename rather than standard output." << endl;
+        cerr << "  -t targetName: Set the targetName used, if not defined use the output filename 'name' and if not defined use the inputFilename 'name'" << endl;
+        cerr << "  -I include_directory: Declare a directory to be added to the includes search pool." << endl;
+        cerr << "  -D varname varvalue: Declare a var used to generate the output file." << endl;
+        cerr << "       varname and varvalue must be made of alpha numerical characters with no spaces." << endl;
+        cerr << "  -listVars : Will list the vars name and value in the standard output." << endl;
+        cerr << "  -showParseTree : Draw the tree obtained while parsing the source" << endl;
+        cerr << "  -c++ : Generate a c++ header file containing the output file stream stored as a char[] variable" << endl;
+        return 0;
+    }
+
+    // Define targetName: if none, get destFilenmae, if none get srcFilename
+    if (targetName.empty()) {
+        if (destFilename.empty()) {
+            targetName = srcFilename;
+        } else {
+            targetName = destFilename;
+        }
+    }
+    // no clean it to have just a descent c var name
+    if (!targetName.empty()) {
+        // trim anything before '/' or '\'
+        targetName = targetName.substr(targetName.find_last_of('/') + 1);
+        targetName = targetName.substr(targetName.find_last_of('\\') + 1);
+
+        // trim anything after '.'
+        targetName = targetName.substr(0, targetName.find_first_of('.'));
+    }
+
+    // Add built in vars
+    time_t endTime = chrono::system_clock::to_time_t(chrono::system_clock::now());
+    std::string endTimStr(ctime(&endTime));
+    vars["_SCRIBE_DATE"] = endTimStr.substr(0, endTimStr.length() - 1);
+
+    // List vars?
+    if (listVars) {
+        cerr << "Vars:" << endl;
+        for (auto v : vars) {
+            cerr << "    " << v.first << " = " << v.second << endl;
+        }
+    }
+
+    // Open up source
+    std::fstream srcStream;
+    srcStream.open(srcFilename, std::fstream::in);
+    if (!srcStream.is_open()) {
+        cerr << "Failed to open source file <" << srcFilename << ">" << endl;
+        return 0;
+    }
+
+    TextTemplate::Pointer scribe(new TextTemplate(srcFilename, config));
+
+    // ready to parse and generate
+    std::ostringstream destStringStream;
+    int numErrors = scribe->scribe(destStringStream, srcStream, vars);
+    if (numErrors) {
+        cerr << "Scribe " << srcFilename << "> failed: " << numErrors << " errors." << endl;
+        return 0;
+    };
+
+
+    if (showParseTree) {
+        int level = 1;
+        cerr << "Config trees:" << std::endl;
+        config->displayTree(cerr, level);
+
+        cerr << srcFilename << " tree:" << std::endl;
+        scribe->displayTree(cerr, level);
+    }
+
+    std::ostringstream targetStringStream;
+    if (makeCPlusPlus) {
+        targetStringStream << "// File generated by Scribe " << vars["_SCRIBE_DATE"] << std::endl;
+
+        targetStringStream << "#ifndef scribe_" << targetName << "_h" << std::endl;
+        targetStringStream << "#define scribe_" << targetName << "_h" << std::endl << std::endl;
+
+        targetStringStream << "const char " << targetName << "[] = {\n\"";
+
+        std::stringstream destStringStreamAgain(destStringStream.str());
+        while (!destStringStreamAgain.eof()) {
+            std::string line;
+            std::getline(destStringStreamAgain, line);
+            targetStringStream << line << " \\n\\\n";
+        }
+        targetStringStream << "\"};" << std::endl << std::endl;
+
+        targetStringStream << "#endif" << std::endl;
+    } else {
+        targetStringStream << destStringStream.str();
+    }
+
+    // Destination stream
+    if (!destFilename.empty()) {
+        std::fstream destFileStream;
+        destFileStream.open(destFilename, std::fstream::out);
+        if (!destFileStream.is_open()) {
+            cerr << "Scribe output file " << destFilename << "> failed to open." << endl;
+            return 0;
+        }
+        
+        destFileStream << targetStringStream.str();
+    } else {
+        cerr << targetStringStream.str();
+    }
+
+    return 0;
+}