adding scribe tool

tools/CMakeLists.txt

@@ -2,3 +2,4 @@
 add_subdirectory(bitstream2json)
 add_subdirectory(json2bitstream)
 add_subdirectory(mtc)
+add_subdirectory(scribe)

tools/scribe/CMakeLists.txt

@@ -0,0 +1,5 @@
+set(TARGET_NAME scribe)
+setup_hifi_project()
+
+# call macro to include our dependency includes and bubble them up via a property on our target
+include_dependency_includes()

tools/scribe/src/TextTemplate.h

@@ -0,0 +1,205 @@
+
+//
+//  TextTemplate.cpp
+//  tools/shaderScribe/src
+//
+//  Created by Sam Gateau on 12/15/2014.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+#ifndef hifi_TEXT_TEMPLATE_H
+#define hifi_TEXT_TEMPLATE_H
+
+#include <list>
+#include <vector>
+#include <map>
+#include <string>
+#include <iostream>
+#include <sstream>
+#include <memory>
+
+class TextTemplate {
+public:
+    typedef std::shared_ptr< TextTemplate > Pointer;
+    typedef std::string String;
+    typedef std::vector< String > StringVector;
+    typedef std::map< String, String > Vars;
+    typedef std::map< String, TextTemplate::Pointer > Includes;
+
+    class Tag {
+    public:
+        enum Type {
+            VARIABLE = 0,
+            COMMAND,
+            REMARK,
+            INVALID = -1,
+        };
+
+        static const char BEGIN = '<';
+        static const char END = '>';
+
+        static const char VAR = '$';
+        static const char COM = '@';
+        static const char REM = '!';
+    };
+
+    class Command {
+    public:
+        typedef std::vector< Command > vector;
+
+        enum Type {
+            VAR = 0,
+            BLOCK,
+            FUNC,
+            ENDFUNC,
+            IFBLOCK,
+            IF,
+            ELIF,
+            ELSE,
+            ENDIF,
+            FOR,
+            ENDFOR,
+            INCLUDE,
+            DEF,
+        };
+
+        Type type;
+        std::vector< String > arguments;
+
+        bool isBlockEnd() {
+            switch (type) {
+                case ENDFUNC:
+                case ENDIF:
+                case ENDFOR:
+                case INCLUDE:
+                case DEF:
+                case VAR:
+                    return true;
+                default:
+                    return false;
+            }
+        }
+    };
+
+    class Block {
+    public:
+        typedef std::shared_ptr<Block> Pointer;
+        typedef std::vector< Block::Pointer > Vector;
+
+        Block::Pointer parent;
+        Command command;
+        Vector blocks;
+        std::stringstream ostr;
+
+        String sourceName;
+
+        Block(const String& sourceFilename) :
+            sourceName(sourceFilename)
+            {}
+
+        static void addNewBlock(const Block::Pointer& parent, const Block::Pointer& block);
+        static const Block::Pointer& getCurrentBlock(const Block::Pointer& block);
+
+        static void displayTree(const Block::Pointer& block, std::ostream& dst, int& level);
+    };
+
+    class Funcs {
+    public:
+        typedef std::map< String, Block::Pointer > map;
+
+        Funcs();
+        ~Funcs();
+
+        const Block::Pointer findFunc(const char* func);
+        const Block::Pointer addFunc(const char* func, const Block::Pointer& root);
+
+        map _funcs;
+    protected:
+    };
+
+    class Config {
+    public:
+        typedef std::shared_ptr< Config > Pointer;
+        typedef bool (*IncluderCallback) (const Config::Pointer& config, const char* filename, String& source);
+
+        Includes        _includes;
+        Funcs           _funcs;
+        std::ostream*   _logStream;
+        int             _numErrors;
+        IncluderCallback _includerCallback;
+        StringVector    _paths;
+
+        Config();
+
+        static const TextTemplate::Pointer addInclude(const Config::Pointer& config, const char* include);
+        const TextTemplate::Pointer findInclude(const char* include);
+
+        void addIncludePath(const char* path);
+
+        void displayTree(std::ostream& dst, int& level) const;
+    };
+
+    static bool loadFile(const Config::Pointer& config, const char* filename, String& source);
+
+    TextTemplate(const String& name, const Config::Pointer& config = Config::Pointer(new Config()));
+    ~TextTemplate();
+
+    // Scibe does all the job of parsing an inout template stream and then gneerating theresulting stream using the vars
+    int scribe(std::ostream& dst, std::istream& src, Vars& vars);
+
+    int parse(std::istream& src);
+    int generate(std::ostream& dst, Vars& vars);
+
+    const Config::Pointer config() { return _config; }
+
+    void displayTree(std::ostream& dst, int& level) const;
+
+protected:
+    Config::Pointer _config;
+    Block::Pointer _root;
+    int _numErrors;
+    bool _steppingStarted;
+
+    bool grabUntilBeginTag(std::istream* str, String& grabbed, Tag::Type& tagType);
+    bool grabUntilEndTag(std::istream* str, String& grabbed, Tag::Type& tagType);
+
+    bool stepForward(std::istream* str, String& grabbed, String& tag, Tag::Type& tagType, Tag::Type& nextTagType);
+
+    bool grabFirstToken(String& src, String& token, String& reminder);
+    bool convertExpressionToArguments(String& src, std::vector< String >& arguments);
+    bool convertExpressionToDefArguments(String& src, std::vector< String >& arguments);
+    bool convertExpressionToFuncArguments(String& src, std::vector< String >& arguments);
+
+    // Filter between var, command or comments
+    const Block::Pointer processStep(const Block::Pointer& block, String& grabbed, String& tag, Tag::Type& tagType);
+    const Block::Pointer processStepVar(const Block::Pointer& block, String& grabbed, String& tag);
+    const Block::Pointer processStepCommand(const Block::Pointer& block, String& grabbed, String& tag);
+    const Block::Pointer processStepRemark(const Block::Pointer& block, String& grabbed, String& tag);
+
+    // Define command
+    const Block::Pointer processStepDef(const Block::Pointer& block, String& grabbed, String& tag);
+
+    // If commands
+    const Block::Pointer processStepCommandIf(const Block::Pointer& block, String& grabbed, String& expression);
+    const Block::Pointer processStepCommandEndIf(const Block::Pointer& block, String& grabbed, String& expression);
+    const Block::Pointer processStepCommandElif(const Block::Pointer& block, String& grabbed, String& expression);
+    const Block::Pointer processStepCommandElse(const Block::Pointer& block, String& grabbed, String& expression);
+
+    // Include command
+    const Block::Pointer processStepInclude(const Block::Pointer& block, String& grabbed, String& tag);
+
+    // Function command
+    const Block::Pointer processStepFunc(const Block::Pointer& block, String& grabbed, String& tag);
+    const Block::Pointer processStepEndFunc(const Block::Pointer& block, String& grabbed, String& tag);
+
+    // Generation
+    int generateTree(std::ostream& dst, const Block::Pointer& block, Vars& vars);
+    int evalBlockGeneration(std::ostream& dst, const Block::Pointer& block, Vars& vars, Block::Pointer& branch);
+
+    // Errors
+    std::ostream& log() { return (* _config->_logStream); }
+    void logError(const Block::Pointer& block, const char* error, ...);
+};
+
+#endif

tools/scribe/src/main.cpp

@@ -0,0 +1,226 @@
+//
+//  main.cpp
+//  tools/shaderScribe/src
+//
+//  Created by Sam Gateau on 12/15/2014.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+
+
+#include "TextTemplate.h"
+
+#include <fstream>
+#include <ctime>
+#include <chrono>
+
+using namespace std;
+
+int main (int argc, char** argv) {
+    // process the command line arguments
+    std::vector< std::string > inputs;
+
+    std::string srcFilename;
+    std::string destFilename;
+    std::string targetName;
+    TextTemplate::Vars vars;
+    
+    std::string lastVarName;
+    bool listVars = false;
+    bool showParseTree = false;
+    bool makeCplusplus = false;
+
+    TextTemplate::Config::Pointer config(new TextTemplate::Config());
+
+    enum Mode {
+        READY = 0,
+        GRAB_OUTPUT,
+        GRAB_VAR_NAME,
+        GRAB_VAR_VALUE,
+        GRAB_INCLUDE_PATH,
+        GRAB_TARGET_NAME,
+        EXIT,
+    } mode = READY;
+    for (int ii = 1; (mode != EXIT) && (ii < argc); ii++) {
+        inputs.push_back(argv[ii]);
+
+        switch (mode) {
+            case READY: {
+                if (inputs.back() == "-o") {
+                    mode = GRAB_OUTPUT;
+                } else if (inputs.back() == "-tn") {
+                    mode = GRAB_TARGET_NAME;
+                } else if (inputs.back() == "-D") {
+                    mode = GRAB_VAR_NAME;
+                } else if (inputs.back() == "-I") {
+                    mode = GRAB_INCLUDE_PATH;
+                } else if (inputs.back() == "-listvars") {
+                    listVars = true;
+                    mode = READY;
+                } else if (inputs.back() == "-showParseTree") {
+                    showParseTree = true;
+                    mode = READY;
+                } else if (inputs.back() == "-c++") {
+                    makeCplusplus = true;
+                    mode = READY;
+                } else {
+                    // just grabbed the source filename, stop parameter parsing
+                    srcFilename = inputs.back();
+                    mode = EXIT;
+                }
+            }
+            break;
+
+            case GRAB_OUTPUT: {
+                destFilename = inputs.back();
+                mode = READY;
+            }
+            break;
+
+            case GRAB_TARGET_NAME: {
+                targetName = inputs.back();
+                mode = READY;
+            }
+            break;
+
+            case GRAB_VAR_NAME: {
+                // grab first the name of the var
+                lastVarName = inputs.back();
+                mode = GRAB_VAR_VALUE;
+            }
+            break;
+            case GRAB_VAR_VALUE: {
+                // and then the value
+                vars.insert(TextTemplate::Vars::value_type(lastVarName, inputs.back()));
+
+                mode = READY;
+            }
+            break;
+
+            case GRAB_INCLUDE_PATH: {
+                config->addIncludePath(inputs.back().c_str());
+                mode = READY;
+            }
+            break;
+                
+            case EXIT: {
+                // THis shouldn't happen
+            }
+            break;
+        }
+    }
+
+    if (srcFilename.empty()) {
+        cerr << "Usage: shaderScribe [OPTION]... inputFilename" << endl;
+        cerr << "Where options include:" << endl;
+        cerr << "  -o filename: Send output to filename rather than standard output." << endl;
+        cerr << "  -tn targetName: Set the targetName used, if not defined use the output filename 'name' and if not defined use the inputFilename 'name'" << endl;
+        cerr << "  -I include_directory: Declare a directory to be added to the includes search pool." << endl;
+        cerr << "  -D varname varvalue: Declare a var used to generate the output file." << endl;
+        cerr << "       varname and varvalue must be made of alpha numerical characters with no spaces." << endl;
+        cerr << "  -listvars : Will list the vars name and value in the standard output." << endl;
+        cerr << "  -showParseTree : Draw the tree obtained while parsing the source" << endl;
+        cerr << "  -c++ : Generate a c++ header file containing the output file stream stored as a char[] variable" << endl;
+        return 0;
+    }
+
+    // Define targetName: if none, get destFilenmae, if none get srcFilename
+    if (targetName.empty()) {
+        if (destFilename.empty()) {
+            targetName = srcFilename;
+        } else {
+            targetName = destFilename;
+        }
+    }
+    // no clean it to have just a descent c var name
+    if (!targetName.empty())
+    {
+        // trim anything before '/' or '\'
+        targetName = targetName.substr(targetName.find_last_of('/') + 1);
+        targetName = targetName.substr(targetName.find_last_of('\\') + 1);
+
+        // trim anything after '.'
+        targetName = targetName.substr(0, targetName.find_first_of('.'));
+    }
+
+    // Add built in vars
+    time_t endTime = chrono::system_clock::to_time_t(chrono::system_clock::now());
+    std::string endTimStr(ctime(&endTime));
+    vars["_SCRIBE_DATE"] = endTimStr.substr(0, endTimStr.length() - 1);
+
+    // List vars?
+    if (listVars) {
+        cerr << "Vars:" << endl;
+        for (auto v : vars) {
+            cerr << "    " << v.first << " = " << v.second << endl;
+        }
+    }
+
+    // Open up source
+    std::fstream srcStream;
+    srcStream.open(srcFilename, std::fstream::in);
+    if (!srcStream.is_open()) {
+        cerr << "Failed to open source file <" << srcFilename << ">" << endl;
+        return 0;
+    }
+
+
+    TextTemplate::Pointer scribe(new TextTemplate(srcFilename, config));
+
+    // ready to parse and generate
+    std::ostringstream destStringStream;
+    int numErrors = scribe->scribe(destStringStream, srcStream, vars);
+    if (numErrors) {
+        cerr << "Scribe " << srcFilename << "> failed: " << numErrors << " errors." << endl;
+        return 0;
+    };
+
+
+    if (showParseTree) {
+        int level = 1;
+        cerr << "Config trees:" << std::endl;
+        config->displayTree(cerr, level);
+
+        cerr << srcFilename << " tree:" << std::endl;
+        scribe->displayTree(cerr, level);
+    }
+
+    std::ostringstream targetStringStream;
+    if (makeCplusplus) {
+        targetStringStream << "// File generated by Scribe " << vars["_SCRIBE_DATE"] << std::endl;
+
+        targetStringStream << "#ifndef scribe_" << targetName << "_h" << std::endl;
+        targetStringStream << "#define scribe_" << targetName << "_h" << std::endl << std::endl;
+
+        targetStringStream << "const char " << targetName << "[] = {\n\"";
+
+        std::stringstream destStringStreamAgain(destStringStream.str());
+        while (!destStringStreamAgain.eof()) {
+            std::string line;
+            std::getline(destStringStreamAgain, line);
+            targetStringStream << line << " \\n\\\n";
+        }
+        targetStringStream << "\"};" << std::endl << std::endl;
+
+        targetStringStream << "#endif" << std::endl;
+    } else {
+        targetStringStream << destStringStream.str();
+    }
+
+    // Destination stream
+    if (!destFilename.empty()) {
+        std::fstream destFileStream;
+        destFileStream.open(destFilename, std::fstream::out);
+        if (!destFileStream.is_open()) {
+            cerr << "Scribe output file " << destFilename << "> failed to open." << endl;
+            return 0;
+        }
+        
+        destFileStream << targetStringStream.str();
+    } else {
+        cerr << targetStringStream.str();
+    }
+
+    return 0;
+}