SAPFOR/Sapfor/_src/Utils/SgUtils.cpp

#include "../Utils/leak_detector.h"

#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <cstdint>

#include <string>
#include <fstream>
#include <sstream>
#include <iostream>

#include <vector>
#include <map>
#include <queue>
#include <set>
#include <utility>
#include <string>
#include <assert.h>
#include <locale>
#include <algorithm>
#include <fcntl.h>
#include <mutex>
#include <thread>
#include <stack>

#ifdef _WIN32
#include <direct.h>
#else
#include <unistd.h> 
#endif

#include "SgUtils.h"
#include "errors.h"
#include "../Sapfor.h"

#include "../DirectiveProcessing/directive_parser.h"
#include "../Distribution/Distribution.h"

#include "../GraphCall/graph_calls.h"
#include "../GraphCall/graph_calls_func.h"
#include "../CreateInterTree/CreateInterTree.h"
#include "../Predictor/PredictScheme.h"
#include "../VisualizerCalls/get_information.h"
#include "../VisualizerCalls/SendMessage.h"
#include "../VerificationCode/verifications.h"
#include "../LoopAnalyzer/loop_analyzer.h"

using std::map;
using std::queue;
using std::multimap;
using std::pair;
using std::tuple;
using std::set;
using std::vector;
using std::string;
using std::make_pair;
using std::make_tuple;
using std::wstring;
using std::stack;

static bool ifIntervalExists(const vector<pair<int, int>> &intervals, const pair<int, int> &toFind, pair<int, int>& inInterval)
{
    bool retVal = false;

    for (auto &elem : intervals)
    {
        if (toFind.first <= elem.first && toFind.second >= elem.second)
        {
            inInterval = elem;
            retVal = true;
            break;
        }
    }

    return retVal;
}

static bool ifDir(const string &line)
{
    if (line.size() >= 10 && (line[0] == '!' || line[0] == 'c'))
    {
        string dir(line.begin() + 1, line.begin() + 1 + 4);
        if (dir == "$spf" || dir == "dvm$")
            return true;
    }
    return false;
}
 
static string replaceTabsToSpaces(const string &in)
{
    string ret = ""; 
    for (auto &elem : in)
    {
        if (elem == '\t')
            ret += "    ";
        else
            ret += elem;
    }
    return ret;
}

static map<string, pair<string, vector<pair<int, int>> > > findIncludes(FILE *currFile)
{
    map<string, pair<string, vector<pair<int, int>> > > includeFiles;
    bool notClosed = false;
    // TODO: extend buff size in dynamic
    char buf[8192];
    int lineBefore = 0;
    while (!feof(currFile))
    {
        char *read = fgets(buf, 8192, currFile);
        if (read)
        {
            const string orig(replaceTabsToSpaces(read));
            string line(read);
            convertToLower(line);
            line = replaceTabsToSpaces(line);

            size_t posF = line.find("include");
            if (posF != string::npos)
            {
                //check if 'inclide' - operator
                for (size_t k = 0; k < posF; ++k)
                {
                    if (line[k] != ' ')
                    {
                        posF = -1;
                        break;
                    }
                }
                if (posF == -1)
                {
                    ++lineBefore;
                    continue;
                }

                posF += sizeof("include") - 1;
                int tok = 0;
                size_t st = string::npos, en = string::npos;

                for (size_t k = posF; k < line.size(); ++k)
                {
                    bool isQuote = line[k] == '\'' || line[k] == '\"';
                    if (isQuote && tok == 1)
                        break;
                    else if (isQuote)
                        tok++;
                    else if (tok == 1 && st == string::npos)
                        st = k;
                    else
                        en = k;
                }
                if (st == string::npos || st >= line.size() || en == string::npos || en >= line.size())
                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

                const string inclName(orig.begin() + st, orig.begin() + en + 1);

                auto toInsert = includeFiles.find(inclName);
                if (toInsert == includeFiles.end())
                {
                    toInsert = includeFiles.insert(toInsert, make_pair(inclName, make_pair(orig, vector<pair<int, int>>())));
                    toInsert->second.second.push_back(make_pair(lineBefore, -1));
                    notClosed = true;
                }
                else
                {
                    toInsert->second.second.push_back(make_pair(lineBefore, -1));
                    notClosed = true;
                }
                //printf("insert %s -> %s\n", inclName.c_str(), line.c_str());
            }
            else if ((line[0] != 'c' && line[0] != '!' && line != "" && line[0] != '\n') || ifDir(line))
            {
                if (notClosed)
                {
                    for (auto &elem : includeFiles)
                        for (auto &pair : elem.second.second)
                            if (pair.second == -1)
                                pair.second = lineBefore + 1;
                    notClosed = false;
                }
            }
        }
        ++lineBefore;
    }

    return includeFiles;
}

static vector<string> findCommentsInIncludes(map<string, pair<string, vector<pair<int, int>> > >& includeFiles)
{
    vector<string> foundComments;

    for (auto& elem : includeFiles)
    {
        FILE* inclFile = fopen(elem.first.c_str(), "r");
        if (inclFile == NULL)
        {
            char buf[256];
            sprintf(buf, "ERROR: Can't open inlcude file %s for read with err %d\n", elem.first.c_str(), errno);
            addToGlobalBufferAndPrint(buf);
            continue;
        }

        char buf[8192];
        int lineBefore = 0;
        string newComment = "";
        while (!feof(inclFile))
        {
            char* read = fgets(buf, 8192, inclFile);
            if (read)
            {
                string orig(replaceTabsToSpaces(read));
                if (orig.size() && (orig[0] == 'C' || orig[0] == 'c'))
                {
                    orig[0] = '!';
                    newComment += orig;
                    continue;
                }

                int k = 0;
                while (k < orig.size())
                {
                    if (orig[k] == ' ')
                        ++k;
                    else
                        break;
                }

                // remove first spaces
                if (k > 0)
                    orig.erase(orig.begin(), orig.begin() + k);

                if (orig.size())
                {
                    if (orig[0] == '!')
                        newComment += read;
                    else if (newComment != "")
                    {
                        foundComments.push_back(newComment);
                        newComment = "";
                    }
                }
                else if (newComment != "")
                {
                    foundComments.push_back(newComment);
                    newComment = "";
                }
            }
        }

        if (newComment != "")
        {
            foundComments.push_back(newComment);
            newComment = "";
        }

        fclose(inclFile);
    }

    return foundComments;
}

static int reverseVar(SgStatement* st)
{
    st->setVariant(-1 * st->variant());
    return st->id();
}

static map<SgStatement*, map<int, set<string>>> insertedIncludes;
static set<SgFile*> genVersionDone;

static set<int> hideUnnecessary(SgFile* file, const string& fileN, const set<string>& moduleDeclsInFiles, bool dontReplaceIncludes)
{
    set<int> changedVars;
    for (SgStatement* st = file->firstStatement(); st; st = st->lexNext())
    {
        if (st->variant() == GLOBAL)
            continue;

        if (dontReplaceIncludes == false)
        {
            if (st->fileName() != fileN || st->getUnparseIgnore())
                if (st->variant() > 0)
                    changedVars.insert(reverseVar(st));
        }
        else
        {
            if (st->getUnparseIgnore())
            {
                if (st->variant() > 0)
                    changedVars.insert(reverseVar(st));
            }
            else if (st->fileName() != fileN)
            {
                if (st->variant() == MODULE_STMT && moduleDeclsInFiles.find(st->fileName()) != moduleDeclsInFiles.end())
                {
                    for (auto mSt = st; mSt != st->lastNodeOfStmt(); mSt = mSt->lexNext())
                        changedVars.insert(reverseVar(mSt));

                    st = st->lastNodeOfStmt();
                    changedVars.insert(reverseVar(st));
                }
            }
        }
    }

    return changedVars;
}

//TODO: read includes and find last lines, all included files
string removeIncludeStatsAndUnparse(SgFile *file, const char *fileName, const char *fout, 
                                    set<string> &allIncludeFiles, bool outFree, 
                                    const map<string, set<string>>& moduleUsesByFile, const map<string, string>& moduleDelcs, 
                                    const map<SgStatement*, vector<SgStatement*>>& exctactedModuleStats,
                                    bool toString, bool renameIncludes, bool dontReplaceIncludes)
{
    removeSpecialCommentsFromProject(file);

    if (genVersionDone.find(file) == genVersionDone.end())
    {
        auto firstStat = file->firstStatement();
        auto com = printVersionAsFortranComm();
        if (com != "")
            firstStat->addComment(com.c_str());
        genVersionDone.insert(file);
    }

    fflush(NULL);

    const set<string> modulesExclude = getExcludedModules();

    set<string> moduleIncudeUses;
    auto itM = moduleUsesByFile.find(fileName);
    if (itM != moduleUsesByFile.end())
    {
        for (auto& elem : itM->second)
        {
            auto it2 = moduleDelcs.find(elem);
            if (it2 == moduleDelcs.end())
            {
                if (modulesExclude.find(elem) != modulesExclude.end())
                    continue;

                __spf_print(1, "can not find module '%s'\n", elem.c_str());
                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
            }
            string lowerInclude = it2->second;
            convertToLower(lowerInclude);

            moduleIncudeUses.insert(lowerInclude);
        }
    }

    int funcNum = file->numberOfFunctions();
    FILE *currFile = fopen(fileName, "r");
    if (currFile == NULL)
    {
        char buf[256];
        sprintf(buf, "ERROR: Can't open file %s for read with err %d\n", fileName, errno);
        addToGlobalBufferAndPrint(buf);
        char buff[1024];
        if (getcwd(buff, 1024) == NULL)
             ;//
        else
             __spf_print(1, "curr path %s\n", buff);
        throw(-1);
    }

    // name -> unparse comment and position
    map<string, pair<string, vector<pair<int, int>> > > includeFiles = findIncludes(currFile);
    fclose(currFile);

    const vector<string> commentsFromIncl = findCommentsInIncludes(includeFiles);

    //add spaces if needed
    if (!outFree)
    {
        for (auto &elem : includeFiles)
        {
            int countSpaces = 0;
            for (int z = 0; z < elem.second.first.size() && elem.second.first[z] == ' '; ++z, ++countSpaces) { }

            if (countSpaces < 6)
            {
                while (countSpaces != 6)
                {
                    elem.second.first.insert(elem.second.first.begin(), ' ');
                    countSpaces++;
                }
            }
        }
    }

    const string fileN = fileName;
    //insert comment
    int lineBefore = -1;

    map<string, set<SgStatement*>> insertedIncludeFiles;
    map<int, vector<pair<int, int>>> placesForInsert;

    set<string> foundForIncludes;
    map<int, int> stIdByLine;

    for (SgStatement* st = file->firstStatement(); st; st = st->lexNext())
    {
        if (st->variant() > 0 && st->lineNumber() > 0 && st->fileName() == string(file->filename()))
        {
            stIdByLine[st->lineNumber()] = st->id();
            vector<SgStatement*> attr = getAttributes<SgStatement*, SgStatement*>(st, set<int>{SPF_ANALYSIS_DIR, SPF_NOINLINE_OP});
            for (auto& elem : attr)
                if (stIdByLine.find(elem->lineNumber()) == stIdByLine.end())
                    stIdByLine[elem->lineNumber()] = st->id();
        }
    }

    auto fileInProject = getAllFilesInProject();

    for (SgStatement *st = file->firstStatement(); st; st = st->lexNext())
    {
        if (st->lineNumber() <= 0 || st->variant() < 0)
            continue;
        string currFileName = st->fileName();

        // included for module use
        if (currFileName != fileN && fileInProject.find(currFileName) != fileInProject.end())
        {
            st = st->lastNodeOfStmt();
            continue;
        }

        if (currFileName != fileN)
        {
            allIncludeFiles.insert(currFileName);
            auto it = includeFiles.find(currFileName);

            if (it != includeFiles.end())
            {
                foundForIncludes.insert(currFileName);
                pair<int, int> lines = make_pair(lineBefore, -1);

                SgStatement *locSt = st->lexNext();
                set<string> nearIncludes;

                while (locSt && (locSt->fileName() != fileN || locSt->lineNumber() <= 0))
                {
                    const string locName = locSt->fileName();

                    if (locName != fileN)
                    {
                        allIncludeFiles.insert(locName);
                        if (locName != currFileName)
                        {
                            auto itNear = includeFiles.find(locName);
                            if (itNear != includeFiles.end())
                            {
                                nearIncludes.insert(locName);
                                foundForIncludes.insert(locName);
                            }
                        }
                    }
                    locSt = locSt->lexNext();
                }

                if (locSt)
                {
                    lines.second = locSt->lineNumber();
                    st = locSt;

                    bool change = true;
                    while (change)
                    {
                        change = false;
                        SgStatement *prev = locSt->lexPrev();
                        if (prev && 
                            (prev->variant() == DVM_PARALLEL_ON_DIR || 
                             prev->variant() == SPF_ANALYSIS_DIR || 
                             prev->variant() == SPF_TRANSFORM_DIR ||
                             prev->variant() == DVM_INTERVAL_DIR || 
                             prev->variant() == ACC_REGION_DIR || 
                             prev->variant() == DVM_REALIGN_DIR ||
                             prev->variant() == DVM_REDISTRIBUTE_DIR))
                        {
                            locSt = prev;
                            change = true;
                        }
                    }
                    placesForInsert[locSt->id()].push_back(lines);
                }
            }           
        }
        else
            lineBefore = st->lineNumber();
    }

    //if not found, it indicates that file contains of other include files
    for (auto& incl : includeFiles)
    {
        if (foundForIncludes.find(incl.first) == foundForIncludes.end())
        {
            for (auto& interval : incl.second.second)
            {
                bool added = false;
                for (int z = interval.second; z > interval.first; z--)
                {
                    auto it = stIdByLine.find(z);
                    if (it != stIdByLine.end())
                    {
                        placesForInsert[it->second].push_back(interval);
                        added = true;
                        break;
                    }
                }

                if (added == false)
                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
            }
            
        }
    }

    for (SgStatement *st = file->firstStatement(); st; st = st->lexNext())
    {
        if (st->variant() < 0)
            continue;

        auto itGlobal = insertedIncludes.find(st);
        if (itGlobal == insertedIncludes.end())
            itGlobal = insertedIncludes.insert(itGlobal, make_pair(st, map<int, set<string>>()));
        map<int, set<string>>& toInsertIncludeComment = itGlobal->second;
        bool needUpdated = false;

        for (auto &it : includeFiles)
        {
            auto foundV = placesForInsert.find(st->id());
            if (foundV != placesForInsert.end())
            {                
                SgStatement* parent = getFuncStat(st, { BLOCK_DATA, MODULE_STMT });
                if(!parent)
                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

                for (auto& interval : foundV->second)
                {
                    pair<int, int> inInterval;
                    if (ifIntervalExists(it.second.second, interval, inInterval) && insertedIncludeFiles[it.first].find(parent) == insertedIncludeFiles[it.first].end())
                    {
                        allIncludeFiles.insert(it.first);
                        insertedIncludeFiles[it.first].insert(parent);

                        string lowerInclude = it.first;
                        convertToLower(lowerInclude);

                        if (moduleIncudeUses.find(lowerInclude) == moduleIncudeUses.end())
                        {
                            if (dontReplaceIncludes == false)
                            {
                                auto itI = toInsertIncludeComment.find(inInterval.first);
                                if (itI == toInsertIncludeComment.end() ||
                                    (itI != toInsertIncludeComment.end() && itI->second.find(it.second.first) == itI->second.end()))
                                {
                                    toInsertIncludeComment[inInterval.first].insert(it.second.first);
                                    needUpdated = true;
                                    __spf_print(1, "  --> restore include '%s' before statement on line %d\n", it.first.c_str(), st->lineNumber());
                                }
                            }
                        }
                    }
                }
            }
        }

        if (needUpdated)
        {
            string inlcude = "";
            for (auto& inclByPos : toInsertIncludeComment)
                for (auto& incl : inclByPos.second)
                    inlcude += (renameIncludes ? renameInclude(incl) : incl);

            if (st->comments())
                st->setComments((inlcude + st->comments()).c_str());
            else
                st->addComment(inlcude.c_str());
        }
    }

    for (auto &inserted : insertedIncludeFiles)
    {
        auto it = includeFiles.find(inserted.first);
        if (it == includeFiles.end())
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

        //TODO: dont work after subroutine copying 
        /*if (inserted.second.size() != it->second.second.size())
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);*/
    }

    set<string> moduleDeclsInFiles;
    for (auto& elem : moduleDelcs)
        moduleDeclsInFiles.insert(elem.second);

    //restore hidden modules
    vector<SgStatement*> toExtract;
    for (auto& mod : exctactedModuleStats)
    {
        auto cp = mod.first->controlParent();
        for (auto& toInsert : mod.second)
        {
            mod.first->insertStmtBefore(*toInsert, *cp);
            toExtract.push_back(toInsert);
        }
    }

    //remove unnecessary
    //XXX: use Sage hack!!
    set<int> changedVars = hideUnnecessary(file, fileN, moduleDeclsInFiles, dontReplaceIncludes);

    set<int> checkPositions;
    for (auto& elem : includeFiles)
        for (auto& interval : elem.second.second)
            checkPositions.insert(interval.second);
    
    // remove comments that were substituted from includes
    for (SgStatement* st = file->firstStatement(); st; st = st->lexNext())
    {
        const char* comments = st->comments();
        if (st->fileName() == fileN && st->variant() > 0 && comments && !dontReplaceIncludes)
        {
            if (checkPositions.find(st->lineNumber()) != checkPositions.end())
            {
                //printf("%s\n", comments);
                string str(comments);
                bool wasDeleted = false;
                for (auto& com : commentsFromIncl)
                {
                    auto it = str.find(com);
                    while (it != string::npos)
                    {
                        wasDeleted = true;
                        str.erase(it, com.size());

                        it = str.find(com);
                    }

                    if (str.size() == 0)
                        break;
                }

                if (wasDeleted)
                {
                    if (str.size() || str.back() != '\n')
                        str += '\n';
                    st->setComments(str.c_str());
                }
            }
        }
    }

    //check for empty
    string comm = "";
    if (file->firstStatement()->comments())
    {
        comm = file->firstStatement()->comments();
        file->firstStatement()->delComments();
    }
    const string tmp = file->firstStatement()->unparse();
    bool isEmpty = (tmp.size() == 0);
    if (comm.size())
        file->firstStatement()->addComment(comm.c_str());

    string strUnparse = "";
    if (toString)
    {
        if (!isEmpty)
            strUnparse = string(file->firstStatement()->unparse());
    }
    else
    {        
        if (!isEmpty)
        {
#ifdef _WIN32
            FILE* fOut;
            errno_t err = fopen_s(&fOut, fout, "w");
#else
            int err = 0;
            FILE* fOut = fopen(fout, "w");
#endif
            if (fOut == NULL)
            {
                if (fout)
                    __spf_print(1, "can not open file to write with name '%s' with error %d\n", fout, err);
                else
                    __spf_print(1, "can not open file to write with name 'NULL'\n");
                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
            }

            file->unparse(fOut);
            fclose(fOut);
        }
    }

    //restore back
    //XXX: use Sage hack!!
    for (SgStatement *st = file->firstStatement(); st; st = st->lexNext())
        if (changedVars.find(st->id()) != changedVars.end())
            st->setVariant(-1 * st->variant());

    for (auto& elem : toExtract)
        elem->extractStmt();

    return strUnparse;
}

static map<SgFile*, map<string, vector<SgSymbol*>>> allCreatedSymbols;

SgSymbol* findSymbolOrCreate(SgFile *file, const string toFind, SgType *type, SgStatement *scope)
{
    auto createdSymbols = allCreatedSymbols.find(file);
    if (createdSymbols == allCreatedSymbols.end())
        createdSymbols = allCreatedSymbols.insert(createdSymbols, make_pair(file, map<string, vector<SgSymbol*>>()));    

    SgSymbol *symb = file->firstSymbol();

    while (symb)
    {
        if (symb->identifier() == toFind)
        {
            bool scopeC = false;
            bool typeC = false;
            if (scope)
                scopeC = (symb->scope() == scope);
            else
                scopeC = true;

            if (type)
                typeC = (symb->type() && symb->type()->equivalentToType(type));
            else
                typeC = true;
            if (scopeC && typeC)
                return symb;
        }
        symb = symb->next();
    }

    auto result = createdSymbols->second.find(toFind);

    if (result == createdSymbols->second.end())
        result = createdSymbols->second.insert(result, make_pair(toFind, vector<SgSymbol*>()));

    SgSymbol *newS = NULL;
    for (auto &symbs : result->second)
    {
        if ((symbs->scope() == scope && scope) || (!symbs->scope() && !scope))
        {
            if (symbs->type() && type)
            {
                if (symbs->type()->equivalentToType(type))
                {
                    newS = symbs;
                    break;
                }
            }
            else if (!symbs->type() && !type)
            {
                newS = symbs;
                break;
            }
        }
    }

    if (newS == NULL)
    {
        newS = new SgSymbol(VARIABLE_NAME, toFind.c_str(), type, scope);
        result->second.push_back(newS);
    }    
    return newS;
}

static string getValue(SgExpression *exp)
{
    if (exp == NULL || exp->variant() == STMT_STR)
        return "";

    string ret = "";
    if (exp->variant() == CONST_REF)
    {
        if (exp->symbol()->identifier())
        {
            ret = "(" + string(exp->symbol()->identifier());
            auto sc = isSgConstantSymb(exp->symbol());
            if (sc && sc->constantValue())
                ret += string("=") + sc->constantValue()->unparse();
            ret += ")";
        }
    }
    else if (exp->symbol())
    {
        if (exp->symbol()->identifier())
            ret = "(" + string(exp->symbol()->identifier()) + ")";
    }
    else if (exp->variant() == INT_VAL)
        ret = "(" + std::to_string(exp->valueInteger()) + ")";
    else if (exp->variant() == DOUBLE_VAL)
        ret = string("(") + (((SgValueExp*)exp)->doubleValue()) + ")";
    else if (exp->variant() == FLOAT_VAL)
        ret = string("(") + (((SgValueExp*)exp)->floatValue()) + ")";
    else if (exp->variant() == ADD_OP)
        ret = "(+)";
    else if (exp->variant() == SUBT_OP)
        ret = "(-)";
    else if (exp->variant() == MULT_OP)
        ret = "(*)";
    else if (exp->variant() == DIV_OP)
        ret = "(/)";
    else if (exp->variant() == MOD_OP)
        ret = "(mod)";
    else if (exp->variant() == EXP_OP)
        ret = "(**)";
    else if (exp->variant() == KEYWORD_VAL)
        ret = "(" + string(((SgKeywordValExp*)exp)->value()) + ")";
    return ret;
}

static void recExpressionPrint(SgExpression *exp, const int lvl, const char *LR, const int currNum, int &allNum)
{
    if (exp)
    {
        SgExpression *lhs = exp->lhs();
        SgExpression *rhs = exp->rhs();
        int lNum, rNum;

        string vCurr = getValue(exp);
        string vL = getValue(lhs);
        string vR = getValue(rhs);

        if (lhs && rhs)
        {
            lNum = allNum + 1;
            rNum = allNum + 2;
            allNum += 2;
            printf("\"%d_%d_%s_%s_%s\" -> \"%d_%d_L_%s_%s\";\n", currNum, lvl, LR, tag[exp->variant()], vCurr.c_str(), lNum, lvl + 1, tag[lhs->variant()], vL.c_str());
            printf("\"%d_%d_%s_%s_%s\" -> \"%d_%d_R_%s_%s\";\n", currNum, lvl, LR, tag[exp->variant()], vCurr.c_str(), rNum, lvl + 1, tag[rhs->variant()], vR.c_str());
        }
        else if (lhs)
        {
            lNum = allNum + 1;
            allNum++;
            printf("\"%d_%d_%s_%s_%s\" -> \"%d_%d_L_%s_%s\";\n", currNum, lvl, LR, tag[exp->variant()], vCurr.c_str(), lNum, lvl + 1, tag[lhs->variant()], vL.c_str());
        }
        else if (rhs)
        {
            rNum = allNum + 1;
            allNum++;
            printf("\"%d_%d_%s_%s_%s\" -> \"%d_%d_R_%s_%s\";\n", currNum, lvl, LR, tag[exp->variant()], vCurr.c_str(), rNum, lvl + 1, tag[rhs->variant()], vR.c_str());
        }
        if (lhs)
            recExpressionPrint(lhs, lvl + 1, "L", lNum, allNum);
        if (rhs)
            recExpressionPrint(rhs, lvl + 1, "R", rNum, allNum);
    }
}

void recExpressionPrint(SgExpression *exp)
{
    printf("digraph G{\n");
    int allNum = 0;
    recExpressionPrint(exp, 0, "L", allNum, allNum);
    if (allNum == 0 && exp)
        printf("\"%d_%d_%s_%s_%s\";\n", allNum, 0, "L", tag[exp->variant()], getValue(exp).c_str());    
    printf("}\n");
    fflush(NULL);
}

void initTags()
{
    for (int i = 0; i < MAXTAGS; i++)
        tag[i] = NULL;

#include "tag.h"
}

void tryToFindPrivateInAttributes(SgStatement *st, set<string> &privates, bool onlyReduction, bool onlyUsers)
{
    set<Symbol*> privatesVars;
    auto foundAttrs = getAttributes<SgStatement*, SgStatement*>(st, set<int>{ SPF_ANALYSIS_DIR, SPF_PARALLEL_DIR });
    if (onlyUsers)
        foundAttrs = filterUserSpf(foundAttrs);

    for (auto &data : foundAttrs)
    {
        if (data)
        {
            Statement *sData = new Statement(data);
            if (!onlyReduction)
                fillPrivatesFromComment(sData, privatesVars);

            // try to find reductions and set its symbols as private in 
            // current loop analysis for correct scalar detection in 
            // tryToFindDependencies() function
            map<string, set<Symbol*>> reductions;
            map<string, set<tuple<Symbol*, Symbol*, int>>> reductionsLoc;

            fillReductionsFromComment(sData, reductions);
            fillReductionsFromComment(sData, reductionsLoc);

            for (auto &redList : reductions)
                privatesVars.insert(redList.second.begin(), redList.second.end());

            for (auto &redLocList : reductionsLoc)
            {
                for (auto &groupList : redLocList.second)
                {
                    privatesVars.insert(std::get<0>(groupList));
                    privatesVars.insert(std::get<1>(groupList));
                }
            }
        }
    }

    for (auto& elem : privatesVars)
        privates.insert(elem->GetOriginal()->identifier());
}

void fillNonDistrArraysAsPrivate(SgStatement *st,
                                 const map<tuple<int, string, string>, pair<DIST::Array*, DIST::ArrayAccessInfo*>> &declaredArrays,
                                 const map<SgStatement*, set<tuple<int, string, string>>> &declaratedArraysSt,
                                 set<string> &privatesVars)
{
    // fill as private all non distributed arrays
    auto it = declaratedArraysSt.find(st);
    if (it != declaratedArraysSt.end())
    {
        for (auto itSet = it->second.begin(); itSet != it->second.end(); ++itSet)
        {
            auto itD = declaredArrays.find(*itSet);
            if (itD != declaredArrays.end())
                if (itD->second.first->IsNotDistribute())
                    privatesVars.insert(itD->second.first->GetShortName());
        }
    }
}

extern  map<tuple<int, string, string>, pair<DIST::Array*, DIST::ArrayAccessInfo*>> declaredArrays;
extern  map<SgStatement*, set<tuple<int, string, string>>> declaratedArraysSt;

DIST::Array* getArrayFromDeclarated(SgStatement *st, const string &arrayName)
{
    DIST::Array *found = NULL;

    auto it = declaratedArraysSt.find(st);
    if (it != declaratedArraysSt.end())
    {
        for (auto itSet = it->second.begin(); itSet != it->second.end() && !found; ++itSet)
        {
            auto itD = declaredArrays.find(*itSet);
            if (itD != declaredArrays.end())
                if (itD->second.first->GetShortName() == arrayName)
                    found = itD->second.first;
        }
    }
    return found;
}

static bool findSymbol(SgExpression *declLst, const string &toFind)
{
    if (declLst == NULL)
        return false;

    stack<SgExpression*> exs;
    exs.push(declLst);

    while (!exs.empty())
    {
        SgExpression* ex = exs.top();
        exs.pop();

        if (ex->variant() == ARRAY_REF || ex->variant() == VAR_REF || ex->variant() == CONST_REF)
        {
            if (ex->symbol()->identifier() == toFind)
                return true;
        }
        else
        {
            if (ex->lhs())
                exs.push(ex->lhs());

            if (ex->rhs())
                exs.push(ex->rhs());
        }
    }
    return false;
}

extern map<string, vector<Messages>> SPF_messages;
SgStatement* declaratedInStmt(SgSymbol *toFind, vector<SgStatement*> *allDecls, bool printInternal, SgStatement* scope)
{
    //need to call this function for MODULE symbols!
    toFind = OriginalSymbol(toFind);
    vector<SgStatement*> inDecl;
    if (toFind->variant() == FUNCTION_NAME && scope == NULL)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    SgStatement *start = scope ? scope : toFind->scope();

    if (start)
    {
        SgStatement *end = start->lastNodeOfStmt();
        const set<int> possibleVars = { VAR_DECL, VAR_DECL_90, ALLOCATABLE_STMT, DIM_STAT, EXTERN_STAT,
                                        COMM_STAT, HPF_TEMPLATE_STAT, DVM_VAR_DECL, STRUCT_DECL };

        while (start != end)
        {
            //error ?
            if (start == NULL)
                break;
            if (start->variant() == INTERFACE_STMT)
                start = start->lastNodeOfStmt();

            //printf("%d on line %d\n", start->variant(), start->lineNumber());
            if (possibleVars.find(start->variant()) != possibleVars.end())
            {
                for (int i = 0; i < 3; ++i)
                {
                    SgExpression *declLst = start->expr(i);
                    if (findSymbol(declLst, toFind->identifier()))
                        inDecl.push_back(start);
                }
            }

            if (!isDVM_stat(start) && !isSPF_stat(start) && isSgExecutableStatement(start))
                break;
            start = start->lexNext();
        }
    }

    /*if (inDecl.size() == 0)
    {
        SgStatement *lowLevelDecl = toFind->declaredInStmt();
        if (lowLevelDecl)
            inDecl.push_back(lowLevelDecl);
    }*/

    if (inDecl.size() == 0)
    {
        if (printInternal)
        {
            __spf_print(1, "can not find declaration for symbol '%s'\n", toFind->identifier());

            auto itM = SPF_messages.find(start->fileName());
            if (itM == SPF_messages.end())
                itM = SPF_messages.insert(itM, make_pair(start->fileName(), vector<Messages>()));

            wstring bufE, bufR;
            __spf_printToLongBuf(bufE, L"Can not find declaration for symbol '%s' in current scope", to_wstring(toFind->identifier()).c_str());
            __spf_printToLongBuf(bufR, R49, to_wstring(toFind->identifier()).c_str());

            itM->second.push_back(Messages(ERROR, toFind->scope()->lineNumber(), bufR, bufE, 1017));
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
        }
        return NULL;
    }

    if (allDecls)
        *allDecls = inDecl;

    //return statement by priority: VAR_DECL, VAR_DECL_90, ALLOCATABLE_STMT, DIM_STAT, other
    if (inDecl.size() > 1)
    {
        for (int i = 0; i < inDecl.size(); ++i)
            if (inDecl[i]->variant() == VAR_DECL || inDecl[i]->variant() == VAR_DECL_90)
                return inDecl[i];

        for (int i = 0; i < inDecl.size(); ++i)
            if (inDecl[i]->variant() == ALLOCATABLE_STMT)
                return inDecl[i];

        for (int i = 0; i < inDecl.size(); ++i)
            if (inDecl[i]->variant() == DIM_STAT)
                return inDecl[i];

        for (int i = 0; i < inDecl.size(); ++i)
            if (inDecl[i]->variant() == COMM_STAT)
                return inDecl[i];

        for (int i = 0; i < inDecl.size(); ++i)
            if (inDecl[i]->variant() == HPF_TEMPLATE_STAT)
                return inDecl[i];

        for (int i = 0; i < inDecl.size(); ++i)
            if (inDecl[i]->variant() == DVM_VAR_DECL)
                return inDecl[i];

        for (int i = 0; i < inDecl.size(); ++i)
            return inDecl[i];

        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
        return NULL;
    }
    else
        return inDecl[0];
}

bool isDVM_stat(SgStatement *st)
{
    if (st == NULL)
        return false;

    bool ret = false;

    const int var = st->variant();

    //for details see dvm_tag.h
    if ((var >= DVM_INTERVAL_DIR && var <= DVM_ENDINTERVAL_DIR) ||
        (var >= DVM_DEBUG_DIR && var <= DVM_TRACEOFF_DIR) ||
        (var >= DVM_PARALLEL_ON_DIR && var <= DVM_SHADOW_DIR) ||
        (var >= DVM_NEW_VALUE_DIR && var <= DVM_POINTER_DIR) ||
        (var >= DVM_TASK_REGION_DIR && var < FORALL_STAT) ||
        (var > FORALL_STAT && var <= DVM_TEMPLATE_DELETE_DIR) ||
        (var >= ACC_REGION_DIR && var <= ACC_ASYNC_OP) ||
        (var == DVM_DISTRIBUTE_DIR) ||
        (var >= HPF_TEMPLATE_STAT && var <= DVM_REDISTRIBUTE_DIR) ||
        (var >= BLOCK_OP && var <= STAGE_OP) || 
        (var >= INDIRECT_OP && var <= SHADOW_NAMES_OP) ||
        (var >= ACC_REGION_DIR && var <= ACC_ASYNC_OP))
            ret = true;
    return ret;
}

bool isSPF_stat(SgStatement *st)
{
    if (st == NULL)
        return false;

    bool ret = false;

    const int var = st->variant();
    //for details see dvm_tag.h
    if (var >= SPF_ANALYSIS_DIR && var <= SPF_PROCESS_PRIVATE_OP)
        ret = true;
    return ret;
}

static map<SgExpression*, string>::const_iterator getStringExpr(SgExpression *ex, map<SgExpression*, string> &collection)
{
    auto it = collection.find(ex);
    if (it == collection.end())
        it = collection.insert(it, make_pair(ex, ex->unparse()));
    return it;
}

bool isEqExpressions(SgExpression *left, SgExpression *right, map<SgExpression*, string> &collection)
{
    if (left == right)
        return true;
    else if (left == NULL || right == NULL)
        return false;
    else
        return getStringExpr(left, collection)->second == getStringExpr(right, collection)->second;
}

static string getCommonName(SgExpression *common)
{
    if (common->symbol())
        return string(common->symbol()->identifier());
    else
        return string("spf_unnamed");
}

void getCommonBlocksRef(map<string, vector<SgExpression*>> &commonBlocks, SgStatement *start, SgStatement *end, const string *nameToSkip)
{
    while (start != end)
    {
        if (start->variant() == CONTAINS_STMT)
            break;

        if (start->variant() == COMM_STAT)
        {
            // fill all common blocks
            for (SgExpression *exp = start->expr(0); exp; exp = exp->rhs())
            {
                const string commonName = getCommonName(exp);

                if (nameToSkip && *nameToSkip == commonName)
                    continue;

                auto it = commonBlocks.find(commonName);
                if (it == commonBlocks.end())
                    it = commonBlocks.insert(it, make_pair(commonName, vector<SgExpression*>()));
                it->second.push_back(exp);
            }
        }
        start = start->lexNext();
    }
}

int checkCommonInMainUnit(SgProject& project, map<string, vector<Messages>>& SPF_messages,
                          const set<DIST::Array*>& arrays, bool forDistrbuted)
{
    int internalExit = 0;

    SgStatement* mainUnit = findMainUnit(&project, SPF_messages);
    checkNull(mainUnit, convertFileName(__FILE__).c_str(), __LINE__);

    map<string, vector<SgExpression*>> commonBlocks;
    getCommonBlocksRef(commonBlocks, mainUnit, mainUnit->lastNodeOfStmt());

    // check array declaration
    for (auto& array : arrays)
    {
        if (array->IsLoopArray() || array->IsTemplate())
            continue;
        if (array->GetLocation().first == DIST::l_COMMON)
        {
            auto nameOfCommon = array->GetLocation().second;
            if (commonBlocks.find(nameOfCommon) == commonBlocks.end())
            {
                auto declPlaces = array->GetDeclInfo();
                for (auto& place : declPlaces)
                {
                    vector<Messages>& currMessages = getObjectForFileFromMap(place.first.c_str(), SPF_messages);
                    if (forDistrbuted)
                    {
                        __spf_print(1, "  ERROR: distributed array '%s' in common block '%s' must have declaration in main unit\n", array->GetShortName().c_str(), nameOfCommon.c_str());

                        wstring messageE, messageR;
                        __spf_printToLongBuf(messageE, L"distributed array '%s' in common block '%s' must have declaration in main unit",
                            to_wstring(array->GetShortName()).c_str(), to_wstring(nameOfCommon).c_str());
                        __spf_printToLongBuf(messageR, R75,
                            to_wstring(array->GetShortName()).c_str(), to_wstring(nameOfCommon).c_str());
                        currMessages.push_back(Messages(ERROR, place.second, messageR, messageE, 1042));
                    }
                    else
                    {
                        __spf_print(1, "  ERROR: array '%s' in common block '%s' must have declaration in main unit for DECLARE insertion\n", array->GetShortName().c_str(), nameOfCommon.c_str());

                        wstring messageE, messageR;
                        __spf_printToLongBuf(messageE, L"array '%s' in common block '%s' must have declaration in main unit for DECLARE insertion",
                            to_wstring(array->GetShortName()).c_str(), to_wstring(nameOfCommon).c_str());
                        __spf_printToLongBuf(messageR, R205,
                            to_wstring(array->GetShortName()).c_str(), to_wstring(nameOfCommon).c_str());
                        currMessages.push_back(Messages(ERROR, place.second, messageR, messageE, 1062));
                    }
                }
                internalExit = 1;
            }
        }
    }

    return internalExit;
}

static SgExpression* isInCommon(const vector<SgExpression*> &commonBlocks, const char *arrayName, int &commonPos)
{
    commonPos = 0;
    for (auto &common : commonBlocks)
    {
        for (SgExpression *currCommon = common->lhs(); currCommon; currCommon = currCommon->rhs())
        {
            if (!strcmp(currCommon->lhs()->symbol()->identifier(), arrayName))
                return common;
            commonPos++;
        }
    }
    return NULL;
}

static map<tuple<string, string, int>, tuple<int, string, string>> tableOfUniqNames;
tuple<int, string, string> getUniqName(const map<string, vector<SgExpression*>> &commonBlocks, SgStatement *decl, SgSymbol *symb)
{
    bool inCommon = false;
    bool needtoCheck = true;
    int commonPos = 0;

    SgExpression *foundCommon = NULL;

    SgStatement *declCP = decl->controlParent();
    // find symbol in parameter list of functions
    if (declCP->variant() == PROC_HEDR || declCP->variant() == FUNC_HEDR)
    {
        if (declCP->variant() == PROC_HEDR)
        {
            const int num = ((SgProcHedrStmt*)declCP)->numberOfParameters();
            for (int i = 0; i < num && needtoCheck; ++i)
                if (!strcmp(((SgProcHedrStmt*)declCP)->parameter(i)->identifier(), symb->identifier()))
                    needtoCheck = false;
        }
        else
        {
            const int num = ((SgFuncHedrStmt*)declCP)->numberOfParameters();
            for (int i = 0; i < num && needtoCheck; ++i)
                if (!strcmp(((SgFuncHedrStmt*)declCP)->parameter(i)->identifier(), symb->identifier()))
                    needtoCheck = false;
        }

        auto declCpCp = declCP->controlParent();
        if (declCpCp && declCpCp->variant() == INTERFACE_STMT && needtoCheck)
            needtoCheck = false;
    }
    else if (declCP->variant() == INTERFACE_STMT)
        needtoCheck = false;

    if (needtoCheck)
    {
        for (auto &common : commonBlocks)
        {
            foundCommon = isInCommon(common.second, symb->identifier(), commonPos);
            if (foundCommon)
            {
                inCommon = true;
                break;
            }
        }
    }

    tuple<int, string, string> retVal;
    if (inCommon)
        retVal = make_tuple(commonPos, string("common_") + getCommonName(foundCommon), string(symb->identifier()));
    else
        retVal = make_tuple(decl->lineNumber(), string(decl->fileName()), string(symb->identifier()));

    auto key = make_tuple(symb->identifier(), decl->fileName(), decl->lineNumber());
    auto it = tableOfUniqNames.find(key);
    if (it == tableOfUniqNames.end())
        tableOfUniqNames.insert(it, make_pair(key, retVal));
    else
    {
        if (it->first != key)
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    }
    return retVal;
}

tuple<int, string, string> getFromUniqTable(SgSymbol *symb)
{
    auto place = declaratedInStmt(symb);
    auto localIt = tableOfUniqNames.find(std::make_tuple(symb->identifier(), place->fileName(), place->lineNumber()));
    if (localIt == tableOfUniqNames.end())
    {
        auto place = declaratedInStmt(symb);
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    }
    
    return localIt->second;
}

SgStatement* findMainUnit(SgProject *proj, map<string, vector<Messages>> &SPF_messages)
{
    SgStatement *mainUnit = NULL;

    vector<SgStatement*> mainUnits;
    for (int i = proj->numberOfFiles() - 1; i >= 0; --i)
    {
        SgFile *file = &(proj->file(i));
        const string file_name = file->filename();

        for (int k = 0; k < file->numberOfFunctions(); ++k)
        {
            SgStatement *func = file->functions(k);
            if (func->variant() == PROG_HEDR && file_name == func->fileName())
            {
                mainUnit = func;
                mainUnits.push_back(func);
            }
        }
    }

    if (mainUnits.size() != 1)
    {
        for (auto& elem : mainUnits)
        {
            vector<Messages>& currMessages = getObjectForFileFromMap(elem->fileName(), SPF_messages);

            wstring messageE, messageR;
            __spf_printToLongBuf(messageE, L"more than one main unit was found");
            __spf_printToLongBuf(messageR, R146);

            currMessages.push_back(Messages(ERROR, elem->lineNumber(), messageR, messageE, 1050));
        }

        if (mainUnits.size() == 0)
        {
            for (int i = proj->numberOfFiles() - 1; i >= 0; --i)
            {
                SgFile* file = &(proj->file(i));
                const char* file_name = file->filename();

                vector<Messages> &currMessages = getObjectForFileFromMap(file_name, SPF_messages);
                wstring messageE, messageR;
                __spf_printToLongBuf(messageE, L"main unit not found");
                __spf_printToLongBuf(messageR, R147);

                currMessages.push_back(Messages(ERROR, file->firstStatement()->lexNext()->lineNumber(), messageR, messageE, 1050));
            }            
        }
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    }
    
    if (!mainUnit->switchToFile())
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    return mainUnit;
}

template<typename IN_TYPE, typename OUT_TYPE>
const vector<OUT_TYPE> getAttributes(IN_TYPE st, const set<int> dataType)
{
    vector<OUT_TYPE> outData;

    for (int i = 0; i < st->numberOfAttributes(); ++i)
    {
        SgAttribute *attr = st->getAttribute(i);
        const int type = st->attributeType(i);
        if (dataType.find(type) != dataType.end())
            if (attr->getAttributeData())
                outData.push_back((OUT_TYPE)(attr->getAttributeData()));
    }

    return outData;
}

template<typename IN_TYPE>
void deleteAttributes(IN_TYPE st, const set<int> dataType)
{
    bool changed = true;
    while (changed)
    {
        changed = false;
        int idxDel = -1;
        for (int i = 0; i < st->numberOfAttributes(); ++i)
        {
            SgAttribute* attr = st->getAttribute(i);
            const int type = st->attributeType(i);
            if (dataType.find(type) != dataType.end())
            {
                if (attr->getAttributeData())
                {
                    idxDel = i;
                    break;
                }
            }
        }

        if (idxDel != -1)
        {
            st->deleteAttribute(idxDel);
            changed = true;
        }
    }
}

template const vector<SgSymbol*> getAttributes(SgSymbol *st, const set<int> dataType);
template const vector<SgSymbol*> getAttributes(SgStatement* st, const set<int> dataType);
template const vector<SgSymbol*> getAttributes(SgExpression* st, const set<int> dataType);
template const vector<char*> getAttributes(SgSymbol *st, const set<int> dataType);
template const vector<int*> getAttributes(SgSymbol* st, const set<int> dataType);
template const vector<SgStatement*> getAttributes(SgStatement *st, const set<int> dataType);
template const vector<SgExpression*> getAttributes(SgExpression *st, const set<int> dataType);
template const vector<SgStatement*> getAttributes(SgExpression *st, const set<int> dataType);
template const vector<DIST::Array*> getAttributes(SgExpression *st, const set<int> dataType);
template const vector<int*> getAttributes(SgExpression *st, const set<int> dataType);
template const vector<FuncInfo*> getAttributes(SgStatement *st, const set<int> dataType);

template void deleteAttributes(SgStatement* st, const set<int> dataType);
template void deleteAttributes(SgExpression* st, const set<int> dataType);

static int isParameterOneOfThese(const string& name, const vector<string>& names)
{
    int parameterOfFunctionItself = -1;
    for (int i = 0; i < names.size(); ++i)
        if (name == names[i])
        {
            parameterOfFunctionItself = i;
            break;
        }
    return parameterOfFunctionItself;
}

#define DEF_T 0
#define USE_T 1
#define DEF_USE_T 2

static void processPartOfAssign(int type, SgExpression *exp, vector<DefUseList> &currentLists, SgFile *file, SgStatement *inStat, 
                                vector<string> &parameterNames, pair<SgSymbol*, string> underCall, int funcPos);

static void addSymbolsToDefUse(const int type, SgExpression *ex, vector<DefUseList> &currentList, 
                               pair<SgSymbol*, string> underCall, int funcPos,
                               SgFile *file, SgStatement *inStat, vector<string>& parameterNames)
{
    if (ex)
    {
        bool next = true;
        if (ex->variant() == VAR_REF || ex->variant() == ARRAY_REF)
        {
            if (ex->symbol())
            {
                string name = string(ex->symbol()->identifier());
                currentList.push_back(DefUseList(type, inStat, ex, file, underCall, make_pair(ex->symbol(), name), funcPos, isParameterOneOfThese(name, parameterNames)));
            }
        }
        else if (ex->variant() == FUNC_CALL)
        {
            underCall = make_pair(ex->symbol(), string(ex->symbol()->identifier()));
            funcPos = 0;

            bool isIntrinsic = isIntrinsicFunctionName(ex->symbol()->identifier());
            for (SgExpression *list = ex->lhs(); list; list = list->rhs(), funcPos++)
            {
                if (!isIntrinsic)
                    processPartOfAssign(DEF_USE_T, list->lhs(), currentList, file, inStat, parameterNames, underCall, funcPos);
                else
                    addSymbolsToDefUse(type, list->lhs(), currentList, underCall, funcPos, file, inStat, parameterNames);
            }
            next = false;
        }

        if (next)
        {
            addSymbolsToDefUse(type, ex->lhs(), currentList, underCall, funcPos, file, inStat, parameterNames);
            addSymbolsToDefUse(type, ex->rhs(), currentList, underCall, funcPos, file, inStat, parameterNames);
        }
    }    
}

static void processPartOfAssign(int type, SgExpression *exp, vector<DefUseList> &currentList, SgFile *file, SgStatement *inStat,
                                vector<string> &parameterNames, pair<SgSymbol*, string> underCall, int funcPos)
{
    if (exp->symbol() && exp->variant() == VAR_REF) // simple assign
    {
        SgSymbol *tmp = exp->symbol();        
        string name = string(tmp->identifier());

        if (type == DEF_USE_T)
        {
            currentList.push_back(DefUseList(DEF_T, inStat, exp, file, underCall, make_pair(tmp, name), funcPos, isParameterOneOfThese(name, parameterNames)));
            currentList.push_back(DefUseList(USE_T, inStat, exp, file, underCall, make_pair(tmp, name), funcPos, isParameterOneOfThese(name, parameterNames)));
        }
        else
            currentList.push_back(DefUseList(DEF_T, inStat, exp, file, underCall, make_pair(tmp, name), funcPos, isParameterOneOfThese(name, parameterNames)));
    }
    //TODO
    else // array ref and etc.
    {
        if (exp->variant() == ARRAY_OP)
        {            
            SgSymbol *tmp = exp->lhs()->symbol();
            string name = string(tmp->identifier());

            if (type == DEF_USE_T)
            {
                currentList.push_back(DefUseList(DEF_T, inStat, exp, file, underCall, make_pair(tmp, name), funcPos, isParameterOneOfThese(name, parameterNames)));
                currentList.push_back(DefUseList(USE_T, inStat, exp, file, underCall, make_pair(tmp, name), funcPos, isParameterOneOfThese(name, parameterNames)));
            }
            else
                currentList.push_back(DefUseList(type, inStat, exp, file, underCall, make_pair(tmp, name), funcPos, isParameterOneOfThese(name, parameterNames)));
            
            addSymbolsToDefUse(1, exp->lhs()->lhs(), currentList, underCall, funcPos, file, inStat, parameterNames);
            addSymbolsToDefUse(1, exp->lhs()->rhs(), currentList, underCall, funcPos, file, inStat, parameterNames);
        }
        else if (exp->variant() == ARRAY_REF)
        {
            SgSymbol *tmp = exp->symbol();            
            string name = string(tmp->identifier());

            if (type == DEF_USE_T)
            {
                currentList.push_back(DefUseList(DEF_T, inStat, exp, file, underCall, make_pair(tmp, name), funcPos, isParameterOneOfThese(name, parameterNames)));
                currentList.push_back(DefUseList(USE_T, inStat, exp, file, underCall, make_pair(tmp, name), funcPos, isParameterOneOfThese(name, parameterNames)));
            }
            else
                currentList.push_back(DefUseList(type, inStat, exp, file, underCall, make_pair(tmp, name), funcPos, isParameterOneOfThese(name, parameterNames)));
            
            addSymbolsToDefUse(1, exp->lhs(), currentList, underCall, funcPos, file, inStat, parameterNames);
            addSymbolsToDefUse(1, exp->rhs(), currentList, underCall, funcPos, file, inStat, parameterNames);
        }
        else
        {
            addSymbolsToDefUse(1, exp->lhs(), currentList, underCall, funcPos, file, inStat, parameterNames);
            addSymbolsToDefUse(1, exp->rhs(), currentList, underCall, funcPos, file, inStat, parameterNames);
        }
    }
}

static void inline addToLists(map<string, vector<DefUseList>> &currentLists, const vector<DefUseList> &tmpList)
{
    for (auto &list : currentLists)
        for (auto &elem : tmpList)
            list.second.push_back(elem);
}

void constructDefUseStep1(SgFile *file, map<string, vector<DefUseList>> &defUseByFunctions, map<string, vector<FuncInfo*>> &allFuncInfo, vector<Messages> &messages)
{
    map<string, vector<FuncInfo*>> curFileFuncInfo;
    vector<LoopGraph*> tmpL;
    map<FuncInfo*, vector<SAPFOR::BasicBlock*>> tmpIR;

    functionAnalyzer(file, curFileFuncInfo, tmpL, messages, tmpIR);

    //functions not found
    if (curFileFuncInfo.size() == 0)
        return;

    auto whereToCopy = allFuncInfo.insert(make_pair(file->filename(), vector<FuncInfo*>()));
    for(auto& it : curFileFuncInfo.begin()->second)
        whereToCopy.first->second.push_back(it);

    map<string, FuncInfo*> funcToFuncInfo = map<string, FuncInfo*>();
    for (int i = 0; i < curFileFuncInfo.begin()->second.size(); ++i)
        funcToFuncInfo.insert(make_pair(curFileFuncInfo.begin()->second[i]->funcName, curFileFuncInfo.begin()->second[i]));

    for (int f = 0; f < file->numberOfFunctions(); ++f)
    {
        //function header and entry points
        map<string, vector<DefUseList>> currentLists;
        vector<DefUseList> tmpList;
        vector<string> parameterNames = vector<string>();

        SgStatement *start = file->functions(f);
        SgStatement *end = start->lastNodeOfStmt();
        int pos;

        auto founded = funcToFuncInfo.find(((SgProgHedrStmt*)start)->nameWithContains());
        start->addAttribute(PROC_CALL, (void*)founded->second, sizeof(FuncInfo));

        SgProgHedrStmt *header = isSgProgHedrStmt(start);
        if (header && start->variant() != PROG_HEDR)
            for (int i = 0; i < header->numberOfParameters(); ++i)
                parameterNames.push_back(header->parameter(i)->identifier());

        currentLists[string(start->symbol()->identifier())] = vector<DefUseList>();
        for (SgStatement *st = start->lexNext(); st != end; st = st->lexNext())
        {            
            if (isSgExecutableStatement(st))
            {
                if (isDVM_stat(st))
                {
                    if (st->variant() != ACC_GET_ACTUAL_DIR && st->variant() != ACC_ACTUAL_DIR)
                        continue;
                }

                tmpList.clear();
                switch (st->variant())
                {
                case ASSIGN_STAT:
                    processPartOfAssign(DEF_T, st->expr(0), tmpList, file, st, parameterNames, make_pair((SgSymbol*)NULL, string("")), -1);
                    addSymbolsToDefUse(USE_T, st->expr(1), tmpList, make_pair((SgSymbol*)NULL, string("")), -1, file, st, parameterNames);

                    addToLists(currentLists, tmpList);                    
                    break;
                case FOR_NODE:
                    if (st->symbol())
                        for (auto &list : currentLists)
                        {
                            string name = string(st->symbol()->identifier());
                            auto vr = new SgVarRefExp(*(st->symbol()));

                            list.second.push_back(DefUseList(1, st, vr, file, make_pair((SgSymbol*)NULL, string("")), make_pair(st->symbol(), name), -1, isParameterOneOfThese(name, parameterNames)));
                            list.second.push_back(DefUseList(0, st, vr, file, make_pair((SgSymbol*)NULL, string("")), make_pair(st->symbol(), name), -1, isParameterOneOfThese(name, parameterNames)));
                        }

                    for (int i = 0; i < 3; ++i)
                        addSymbolsToDefUse(USE_T, st->expr(i), tmpList, make_pair((SgSymbol*)NULL, string("")), -1, file, st, parameterNames);
                    addToLists(currentLists, tmpList);
                    break;                
                case READ_STAT:
                    {
                        SgInputOutputStmt *io = isSgInputOutputStmt(st);
                        auto list = io->itemList();
                        int pos = 0;
                        while (list)
                        {
                            if (list->lhs())
                                processPartOfAssign(DEF_T, list->lhs(), tmpList, file, st, parameterNames, make_pair(io->symbol(), string("READ")), pos);
                            pos++;
                            list = list->rhs();
                        }
                    }
                    addToLists(currentLists, tmpList);
                    break;
                case PROC_STAT:
                    pos = 0;
                    for (SgExpression *ex = st->expr(0); ex; ex = ex->rhs(), pos++)
                        processPartOfAssign(DEF_USE_T, ex->lhs(), tmpList, file, st, parameterNames, make_pair(st->symbol(), string(st->symbol()->identifier())), pos);
                    addToLists(currentLists, tmpList);
                    break;
                case ENTRY_STAT:
                    currentLists[string(st->symbol()->identifier())] = vector<DefUseList>();
                    break;
                case LOGIF_NODE:
                case CONT_STAT:
                case IF_NODE:
                case WHILE_NODE:
                case DO_WHILE_NODE:
                case GOTO_NODE:
                case STOP_STAT:
                case RETURN_STAT:
                case RETURN_NODE:
                case ELSEIF_NODE:
                case ARITHIF_NODE:
                case WHERE_NODE:
                case WHERE_BLOCK_STMT:
                case SWITCH_NODE:
                case CASE_NODE:
                case BREAK_NODE:
                case EXIT_STMT:
                case ASSGOTO_NODE:
                case COMGOTO_NODE:
                case WRITE_STAT:
                case PRINT_STAT:
                default:
                    for (int i = 0; i < 3; ++i)
                        addSymbolsToDefUse(USE_T, st->expr(i), tmpList, make_pair((SgSymbol*)NULL, string("")), -1, file, st, parameterNames);
                    addToLists(currentLists, tmpList);
                    break;
                }
            }
        }

        for (auto &list : currentLists)
        {
            auto it = defUseByFunctions.find(list.first);
            if (it == defUseByFunctions.end())
                it = defUseByFunctions.insert(it, make_pair(list.first, vector<DefUseList>()));
        }

        for (auto &list : currentLists)
        {
            auto it = defUseByFunctions.find(list.first);
            for (auto &elem : list.second)
                it->second.push_back(elem);
        }
    }
}

#undef DEF_T
#undef USE_T
#undef DEF_USE_T

extern map<string, vector<DefUseList>> defUseByFunctions;
set<string> getAllDefVars(const string &funcName)
{
    set<string> retVal;
    for (auto &elem : defUseByFunctions[funcName])
        if (elem.isDef())
            retVal.insert(elem.getVar());
    return retVal;
}

set<string> getAllUseVars(const string &funcName)
{
    set<string> retVal;

    for (auto &elem : defUseByFunctions[funcName])
        if (elem.isUse())
            retVal.insert(elem.getVar());

    return retVal;
}

const vector<DefUseList>& getAllDefUseVarsList(const string &funcName) 
{
    return defUseByFunctions[funcName]; 
}

const vector<DefUseList> getAllDefUseVarsList(const string &funcName, const string varName) 
{
    vector<DefUseList> retVal;
    for (auto &elem : defUseByFunctions[funcName])
        if (elem.getVar() == varName)
            retVal.push_back(elem);    

    return retVal;
}

map<SgStatement*, vector<DefUseList>> createDefUseMapByPlace()
{
    map<SgStatement*, vector<DefUseList>> retVal;

    for (auto &byFunc : defUseByFunctions)
        for (auto &elem : byFunc.second)
            retVal[elem.getPlace()].push_back(elem);

    return retVal;
}

int printDefUseSets(const char *fileName, const map<string, vector<DefUseList>> &defUseLists)
{
    FILE *file = fopen(fileName, "w");
    if (file == NULL)
    {
        __spf_print(1, "can not open file %s\n", fileName);
        return -1;
    }
        
    for (auto &elem : defUseLists)
    {
        fprintf(file, "*** Function %s\n", elem.first.c_str());
        for (auto &inList : elem.second)
            inList.print(file, true);
        fprintf(file, "\n");
    }

    fclose(file);
    return 0;
}

static bool isDefUseVar(const int paramPosition, const string &funcName, map<string, vector<DefUseList>> &defUseByFunctions, bool defined)
{
    auto founded = defUseByFunctions.find(funcName);
    if(founded == defUseByFunctions.end())
        return true; //No information. Argument can be defined or used.

    vector<DefUseList>& curDefUse = founded->second;
    bool isDefUse = false;

    for (int i = 0; i < curDefUse.size(); ++i)
    {
        if (paramPosition == curDefUse[i].getParameterPositionInFunction())
        {
            if (defined && curDefUse[i].isDef())
            {
                isDefUse = true;
                break;
            }
            else if(!defined && curDefUse[i].isUse())
            {
                isDefUse = true;
                break;
            }
            else
            {
                const string calledFuncName = curDefUse[i].getParamOfFunction();
                if (!calledFuncName.empty())
                {
                    isDefUse = isDefUseVar(curDefUse[i].getParameterPosition(), calledFuncName, defUseByFunctions, defined);
                    if (isDefUse)
                        break;
                }
            }
        }
    }
    return isDefUse;
}

static SgExpression* makeList(const vector<SgExpression*> &list)
{
    if (list.size() == 0)
        return NULL;
    SgExpression *out = new SgExpression(EXPR_LIST);
    SgExpression *ret = out;
    for (int i = 0; i < list.size(); ++i)
    {
        out->setLhs(list[i]);
        if (i != list.size() - 1)
        {
            out->setRhs(new SgExpression(EXPR_LIST));
            out = out->rhs();
        }
        else
            out->setRhs(NULL);
    }

    return ret;
}

static map<SgStatement*, pair<SgExpression*, SgExpression*>> buildLists(const vector<DefUseList> &list)
{
    map<SgStatement*, pair<vector<SgExpression*>, vector<SgExpression*>>> out;
    for (auto &elem : list)
    {
        if (elem.isDef())
            out[elem.getPlace()].first.push_back(elem.getExpr());
        if (elem.isUse())
            out[elem.getPlace()].second.push_back(elem.getExpr());
    }

    map<SgStatement*, pair<SgExpression*, SgExpression*>> ret;
    for (auto &elem : out)
    {
        ret[elem.first].first = makeList(elem.second.first);
        ret[elem.first].second = makeList(elem.second.second);
    }

    return ret;
}

void constructDefUseStep2(SgFile *file, map<string, vector<DefUseList>> &defUseByFunctions)
{

    for (int f = 0; f < file->numberOfFunctions(); ++f)
    {
        SgProgHedrStmt *header = isSgProgHedrStmt(file->functions(f));
        if (header == NULL || header->variant() == PROG_HEDR)
            continue;

        for (int i = 0; i < header->numberOfParameters(); ++i)
        {
            int currAttr = header->parameter(i)->attributes();
            if ((currAttr & OUT_BIT) == 0 && (currAttr & INOUT_BIT) == 0 && (currAttr & IN_BIT) == 0)
            {
                bool isDef = isDefUseVar(i, header->symbol()->identifier(), defUseByFunctions, true);
                bool isUse = isDefUseVar(i, header->symbol()->identifier(), defUseByFunctions, false);
                if(isDef && isUse)
                    header->parameter(i)->setAttribute(INOUT_BIT);
                else if (isDef)
                    header->parameter(i)->setAttribute(OUT_BIT);
                else
                    header->parameter(i)->setAttribute(IN_BIT);
            }
            //TODO: test and replace from defUseVar() in defUse.cpp
            /*auto funcList = defUseByFunctions[header->symbol()->identifier()];
            auto toAdd = buildLists(funcList);
            for (auto &elem : toAdd)
            {                
                elem.first->addAttribute(DEFINEDLIST_ATTRIBUTE, (void*)elem.second.first, 0);
                elem.first->addAttribute(USEDLIST_ATTRIBUTE, (void*)elem.second.second, 0);
            }*/
        }
    }
}

int printCommonBlocks(const char *fileName, const map<string, CommonBlock*> &commonBlocks)
{
    FILE *file = fopen(fileName, "w");
    if (file == NULL)
    {
        __spf_print(1, "can not open file %s\n", fileName);
        return -1;
    }

    for (auto &commonBlock : commonBlocks)
    {
        fprintf(file, "*** COMMON BLOCK '%s'\n", commonBlock.first.c_str());
        commonBlock.second->print(file);
        fprintf(file, "\n");
    }

    fclose(file);
    return 0;
}

// CommonBlock::
Variable* CommonBlock::hasVariable(const string &name, const varType type, const int position)
{
    for (auto &variable : variables)
        if (variable->getName() == name && variable->getType() == type && variable->getPosition() == position)
            return variable;

    return NULL;
}

Variable* CommonBlock::hasVariable(SgSymbol *symbol, const varType type, const int position)
{
    return hasVariable(string(symbol->identifier()), type, position);
}

const vector<const Variable*> CommonBlock::getVariables(SgFile *file, SgStatement *function) const 
{
    return getVariables(string(file->filename()), string(function->symbol()->identifier()));
}

const vector<const Variable*> CommonBlock::getVariables(const string &file, const string &function) const
{
    vector<const Variable*> retVariables;
    
    for (auto &variable : variables)
        if (variable->hasUse(file, function))
            retVariables.push_back(variable);
    
    return retVariables;
}

const vector<const Variable*> CommonBlock::getVariables(int position) const
{
    vector <const Variable*> retVariables;

    auto it = groupedVars.find(position);
    if (it != groupedVars.end())
        for (auto &variable : it->second)
            retVariables.push_back(variable);

    return retVariables;
}

static void findDeclType(SgExpression *ex, varType &type, const string &toFind)
{
    if (ex)
    {
        if (ex->symbol() && ex->symbol()->identifier() == toFind)
        {
            switch (ex->variant())
            {
            case VAR_REF:
                if (type != ARRAY)
                    type = SCALAR;
                break;
            case ARRAY_REF:
                type = ARRAY;
                break;
            case INT_VAL:
            case FLOAT_VAL:
            case DOUBLE_VAL:
            case BOOL_VAL:
            case CHAR_VAL:
            case STRING_VAL:
            case CONST_REF:
                type = CONST;
                break;
            case COMM_LIST:
                break;
            default:
                type = ANOTHER;
                break;
            }
        }

        findDeclType(ex->lhs(), type, toFind);
        findDeclType(ex->rhs(), type, toFind);
    }
}

void CommonBlock::addVariables(SgFile *file, SgStatement *function, const vector<pair<SgSymbol*, int>> &newVariables)
{
    for (auto &varPair : newVariables)
    {
        vector<SgStatement*> allDecls;
        declaratedInStmt(varPair.first, &allDecls);
        varType type = ANOTHER;

        for (auto& decl : allDecls)
            for (int i = 0; i < 3; ++i)
                findDeclType(decl->expr(i), type, varPair.first->identifier());

        Variable *exist = hasVariable(varPair.first, type, varPair.second);
        if (exist)
            exist->addUse(file, function, varPair.first);
        else
        {
            variables.push_back(new Variable(file, function, varPair.first, string(varPair.first->identifier()), type, varPair.second));

            auto it = groupedVars.find(varPair.second);
            if (it == groupedVars.end())
                it = groupedVars.insert(it, make_pair(varPair.second, vector<Variable*>()));

            it->second.push_back(variables.back());
        }
    }
}

void CommonBlock::print(FILE *fileOut) const
{
    fprintf(fileOut, "  [POSITION], [VARIABLE NAME], [TYPE]: \n");
    for (auto &var: variables)
    {        
        var->print(fileOut);
        fprintf(fileOut, "      USE in [FILE, FUNC / BLOCK_DATA]:");

        for (auto &use : var->getAllUse())
            fprintf(fileOut, " [%s, %s]", use.getFileName().c_str(), 
                                          use.isBlockDataUse() ? ("BD_" + use.getFunctionName()).c_str() : ("F_" + use.getFunctionName()).c_str());
        fprintf(fileOut, "\n");
    }
}

DIST::Array* CommonBlock::getFirstSynonym(DIST::Array* array) const
{
    if (array->GetLocation().first != DIST::l_COMMON)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    for (auto& var : variables)
    {
        if (var->getName() == array->GetShortName())
        {
            int pos = var->getPosition();
            auto it = groupedVars.find(pos);
            if (it == groupedVars.end())
                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

            if (it->second.size() == 1)
                return array;
            else
            {
                const string currFile = current_file->filename();
                auto s = it->second[0]->getSymbol();
                if (SgFile::switchToFile(it->second[0]->getDeclarated()->fileName()) == -1)
                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

                string arrayName = string(s->identifier());
                DIST::Array* firstArray = getArrayFromDeclarated(declaratedInStmt(s), arrayName);

                checkNull(firstArray, convertFileName(__FILE__).c_str(), __LINE__);

                if (SgFile::switchToFile(currFile) == -1)
                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
                return firstArray;
            }            
        }
    }

    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
}

// END of CommonBlock::

void groupDeclarations(SgFile *file)
{
    for (int i = 0; i < file->numberOfFunctions(); ++i)
    {
        SgStatement *func = file->functions(i);
        SgStatement *lastNode = func->lastNodeOfStmt();

        map<SgType*, pair<vector<SgStatement*>, vector<SgExpression*>>> refs;
        
        vector<SgStatement*> toDel;
        for (auto st = func; st != lastNode; st = st->lexNext())
        {
            if (isSgExecutableStatement(st))
                break;
            
            if (st->variant() == VAR_DECL)
            {
                auto varDecl = isSgVarDeclStmt(st);
                SgExpression *varList = varDecl->varList();
                
                int count = 0;
                for (auto lst = varList; lst; lst = lst->rhs())
                    count++;

                vector<SgExpression*> notMoved;

                int countOfMoved = 0;
                for (auto lst = varList; lst; lst = lst->rhs())
                {
                    SgExpression *var = lst->lhs();
                    if (var->variant() == VAR_REF)
                    {
                        auto type = var->symbol()->type();
                        if (type && isSgArrayType(type) == NULL && type->variant() != T_STRING)
                        {
                            countOfMoved++;                            
                            refs[type].second.push_back(var);

                            SgStatement *prev = st->lexPrev();
                            while (prev && prev->variant() == SPF_ANALYSIS_DIR)
                            {
                                refs[type].first.push_back(prev->extractStmt());
                                prev = st->lexPrev();
                            }
                        }
                        else
                            notMoved.push_back(var);
                    }
                    else
                        notMoved.push_back(var);
                }
                                
                if (countOfMoved != count)
                {
                    if (countOfMoved != 0)
                    {
                        SgExpression *varList = new SgExpression(EXPR_LIST);
                        SgExpression *pVar = varList;

                        for (int z = 0; z < notMoved.size(); z++)
                        {                            
                            pVar->setLhs(notMoved[z]);
                            if (z != notMoved.size() - 1)
                            {
                                pVar->setRhs(new SgExpression(EXPR_LIST));
                                pVar = pVar->rhs();
                            }
                        }                        
                        st->setExpression(0, *varList);
                    }
                }
                else
                    toDel.push_back(st);
            }
        }

        for (auto st = func; st != lastNode; st = st->lexNext())
        {
            if (isSgExecutableStatement(st->lexNext()))
            {
                for (auto &newDecl : refs)
                {
                    SgVarDeclStmt *newDeclSt = newDecl.second.second[0]->symbol()->makeVarDeclStmt();
                    for (int z = 1; z < newDecl.second.second.size(); ++z)
                        newDeclSt->addVar(*newDecl.second.second[z]);
                    st->insertStmtAfter(*newDeclSt, *func);

                    for (auto &spf : newDecl.second.first)
                        st->insertStmtAfter(*spf, *func);                    
                }                
                break;
            }
        }
          
        //TODO: move comments
        for (auto &elem : toDel)
            elem->deleteStmt();
    }
}

bool ifSymbolExists(SgFile *file, const string &symbName)
{
    if (SgFile::switchToFile(file->filename()) != -1)
    {
        SgSymbol *symb = file->firstSymbol();

        while (symb)
        {
            if (symb->identifier() == symbName)
                return true;

            symb = symb->next();
        }
    }
    else
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    return false;
}

int checkSymbNameAndCorrect(const string& symbName, int complite)
{
    set<string> existedSymbols;
    //if (existedSymbols.size() == 0)
    {
        SgFile* oldFile = current_file;
        for (int i = 0; i < CurrentProject->numberOfFiles(); ++i)
        {
            SgFile* file = &(CurrentProject->file(i));
            SgSymbol* s = file->firstSymbol();
            while (s)
            {
                existedSymbols.insert(s->identifier());
                s = s->next();
            }
        }

        if (SgFile::switchToFile(oldFile->filename()) == -1)
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    }

    string base = symbName;
    int retNum = complite;
    while (existedSymbols.find(base + std::to_string(retNum)) != existedSymbols.end())
        retNum++;
    //existedSymbols.insert(retName);

    return retNum;
}

string checkSymbNameAndCorrect(const string& symbName, const string complite)
{
    set<string> existedSymbols;
    //if (existedSymbols.size() == 0)
    {
        string oldFileName = current_file->filename();
        for (int i = 0; i < CurrentProject->numberOfFiles(); ++i)
        {
            SgFile* file = &(CurrentProject->file(i));
            SgSymbol* s = file->firstSymbol();
            while (s)
            {
                existedSymbols.insert(s->identifier());
                s = s->next();
            }
        }

        if (SgFile::switchToFile(oldFileName) == -1)
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    }

    string retName = symbName;
    while (existedSymbols.find(retName) != existedSymbols.end())
        retName += complite;
    //existedSymbols.insert(retName);

    return retName;
}

const CommonBlock* isArrayInCommon(const map<string, CommonBlock*> &commonBlocks, const DIST::Array *array)
{
    for (auto &commonBlockPair : commonBlocks)
        for (auto &variable : commonBlockPair.second->getVariables())
            if (variable->getName() == array->GetShortName() && variable->getType() == ARRAY && array->GetLocation().first == DIST::l_COMMON)
                return commonBlockPair.second;

    return NULL;
}

static void fillArraysFromDirsRec(SgExpression *ex, vector<DIST::Array*> &toAdd)
{
    if (ex)
    {
        if (ex->variant() == ARRAY_REF)
        {
            auto attributes = getAttributes<SgExpression*, DIST::Array*>(ex, { ARRAY_REF });
            toAdd.insert(toAdd.end(), attributes.begin(), attributes.end());
        }

        if (ex->lhs())
            fillArraysFromDirsRec(ex->lhs(), toAdd);
        if (ex->rhs())
            fillArraysFromDirsRec(ex->rhs(), toAdd);
    }
}

vector<DIST::Array*> fillArraysFromDir(Statement *st)
{
    vector<DIST::Array*> retVal;
    for (int z = 0; z < 3; ++z)
        fillArraysFromDirsRec(st->GetOriginal()->lexPrev()->expr(z), retVal);
    return retVal;
}

template<typename objT>
objT& getObjectForFileFromMap(const char *fileName, map<string, objT> &mapObject)
{
    auto it = mapObject.find(fileName);
    if (it == mapObject.end())
        it = mapObject.insert(it, std::make_pair(fileName, objT()));
    return it->second;
}
template vector<SpfInterval*>& getObjectForFileFromMap(const char *fileName, map<string, vector<SpfInterval*>>&);
template PredictorStats& getObjectForFileFromMap(const char *fileName, map<string, PredictorStats>&);

void printSymbolTable(SgFile *file, string filter, const set<int>& vars)
{
    for (auto s = file->firstSymbol(); s; s = s->next())
    {
        if (vars.size() != 0)
            if (vars.find(s->variant()) == vars.end())
                continue;

        auto t = s->type();
        bool need = true;
        if (filter != "")
            if (filter != s->identifier())
                need = false;
        if (need)
        {
            int line = s->scope() ? s->scope()->lineNumber() : -1;
            printf("[%d] '%s' type %d ('%s'), location %d line, variant %d, addr %p\n", s->id(), s->identifier(), t ? t->variant() : -1, t ? tag[t->variant()] : "", line, s->variant(), s->thesymb);
        }
    }
}

static bool checkAdd(const int var, const set<int> &additional)
{
    if (additional.size())
    {
        if (additional.find(var) == additional.end())
            return true;
        else
            return false;
    }
    else
        return true;
}

SgStatement* getFuncStat(SgStatement *st, const set<int> additional)
{
    if (!st)
        return NULL;

    SgStatement *iterator = st;
    while (iterator && !isSgProgHedrStmt(iterator) && checkAdd(iterator->variant(), additional))
        iterator = iterator->controlParent();

    if (!iterator)
        checkNull(iterator, convertFileName(__FILE__).c_str(), __LINE__);
    return iterator;
}

SgStatement* duplicateProcedure(SgStatement *toDup, const string *newName, bool withAttributes, bool withComment, bool withSameLines, bool dontInsert)
{
    if (toDup == NULL)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    if (isSgProgHedrStmt(toDup) == NULL)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    SgStatement* global = current_file->firstStatement();
    if (global->variant() != GLOBAL)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    SgSymbol* orig = toDup->symbol();    
    SgSymbol* copied = &orig->copySubprogram(*global);

    //XXX: remove all extra functions
    if (global->lexNext()->symbol()->identifier() != string(copied->identifier()))
    {
        vector<SgStatement*> toDel;
        SgStatement* st = global->lexNext();
        if (!isSgProgHedrStmt(st))
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

        while (st)
        {
            if (st->symbol() == copied)
                break;
            toDel.push_back(st);
            st = st->lastNodeOfStmt()->lexNext();
        }

        if (st == NULL)
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
        if (!isSgProgHedrStmt(st))
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
        if (st->symbol()->identifier() != string(copied->identifier()))
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
        for (auto& elem : toDel)
            elem->extractStmt();

        if (global->lexNext()->symbol()->identifier() != string(copied->identifier()))
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    }

    //move 
    SgStatement* toMove = NULL;
    if (dontInsert)
        toMove = global->lexNext()->extractStmt();
    else
    {
        if (toDup->controlParent())
        {
            toMove = global->lexNext()->extractStmt();
            toDup->insertStmtBefore(*toMove, *toDup->controlParent());
        }
        else
            toMove = global->lexNext();
    }

    //change name
    if (newName)
        copied->changeName(newName->c_str());
    
    // set line numbers, pointer to attributes, comments and unparseIgnore status
    for (auto origStat = toDup, copyStat = toMove;
         origStat != toDup->lastNodeOfStmt()->lexNext();
         origStat = origStat->lexNext(), copyStat = copyStat->lexNext())
    {
        if (copyStat->variant() != origStat->variant())
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

        if (withSameLines)
        {
            copyStat->setlineNumber(origStat->lineNumber());
            BIF_FILE_NAME(copyStat->thebif) = BIF_FILE_NAME(origStat->thebif);
        }

        if (withAttributes)
            if (origStat->numberOfAttributes() > 0)
                copyStat->addAttributeTree(origStat->getAttribute(0));

        if (withComment)
        {
            if (origStat->comments())
                copyStat->setComments(origStat->comments());
        }
        copyStat->setUnparseIgnore(origStat->getUnparseIgnore());
    }

    return toMove;
}

SgExpression* makeExprList(const vector<SgExpression*>& items, bool withSort)
{
    SgExpression* list = NULL;
    if (items.size() == 0)
        return list;
    list = new SgExpression(EXPR_LIST);

    vector<SgExpression*> newItems;
    if (withSort)
    {
         multimap<string, SgExpression*> sorted;
         int tmpVal = 0;
         for (auto& elem : items)
         {
             if (elem->variant() == VAR_REF || elem->variant() == ARRAY_REF || elem->variant() == CONST_REF)
                 sorted.insert(make_pair(elem->unparse(), elem));
             else if (elem->lhs() && elem->lhs()->variant() == VAR_REF || elem->lhs()->variant() == ARRAY_REF)
                 sorted.insert(make_pair(elem->lhs()->unparse(), elem));
             else
                 sorted.insert(make_pair(std::to_string(tmpVal++), elem));
         }

         for (auto& elem : sorted)
             newItems.push_back(elem.second);
         std::reverse(newItems.begin(), newItems.end());
    }
    else
        newItems = items;

    for (int z = 0; z < newItems.size(); ++z)
    {
        if (z == 0)
            list->setLhs(newItems[z]);
        else
        {
            SgExpression* tmp = new SgExpression(EXPR_LIST);
            tmp->setLhs(newItems[z]);
            tmp->setRhs(list);
            list = tmp;
        }
    }

    return list;
}

static bool sortByName(const pair<SgSymbol*, SgExpression*>& l, const pair<SgSymbol*, SgExpression*>& r)
{
    return string(l.first->identifier()) < r.first->identifier();
}

//if curr == NULL -> no insertion
//all vars in sIn must have the same type 
SgStatement* makeDeclaration(SgStatement* curr, const vector<SgSymbol*>& sIn, vector<SgExpression*>* inits)
{
    if (sIn.size() == 0)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    SgStatement* place = curr;
    SgStatement* scope = NULL;

    vector<pair<SgSymbol*, SgExpression*>> s;
    if (place)
    {
        //check same type
        map<string, map<string, vector<SgSymbol*>>> groups;
        for (auto& toDec : sIn)
        {
            string newDecl = makeDeclaration(NULL, { toDec })->unparse();
            auto it = newDecl.find("::");
            if (it == string::npos)
                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

            groups[newDecl.substr(0, it)][toDec->identifier()].push_back(toDec);
        }

        if (groups.size() > 1)
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

        while (isSgProgHedrStmt(place) == NULL && place->variant() != BLOCK_DATA && place->variant() != MODULE_STMT)
            place = place->controlParent();
        scope = place;

        set<string> declared;
        while (isSgExecutableStatement(place) == NULL && place != scope->lastNodeOfStmt())
        {
            if (place->variant() == VAR_DECL || place->variant() == VAR_DECL_90)
            {
                SgExpression* ex = place->expr(0);
                while (ex)
                {
                    if (ex->lhs() && ex->lhs()->symbol())
                        declared.insert(ex->lhs()->symbol()->identifier());
                    ex = ex->rhs();
                }
            }
            place = place->lexNext();
        }

        for (int z = 0; z < sIn.size(); ++z)
            if (declared.find(sIn[z]->identifier()) == declared.end())
                s.push_back(make_pair(sIn[z], (inits ? (*inits)[z] : NULL)));
    }
    else
    {
        for (int z = 0; z < sIn.size(); ++z)
            s.push_back(make_pair(sIn[z], (inits ? (*inits)[z] : NULL)));
    }

    if (s.size() == 0)
        return NULL;

    sort(s.begin(), s.end(), sortByName);

    SgVarDeclStmt* decl = s[0].first->makeVarDeclStmt();
    if (s[0].second)
        decl->expr(0)->setLhs(new SgExpression(ASSGN_OP, new SgVarRefExp(s[0].first), s[0].second));

    for (int z = 1; z < s.size(); ++z)
    {
        auto tmpDecl = s[z].first->makeVarDeclStmt();
        if (s[z].second)
            decl->addVar(*new SgExpression(ASSGN_OP, new SgVarRefExp(s[z].first), s[z].second));
        
        else
            decl->addVar(*tmpDecl->expr(0)->lhs());
    }

    if (place)
    {
        decl->setFileName(place->fileName());
        decl->setFileId(place->getFileId());
        decl->setProject(place->getProject());
        decl->setlineNumber(getNextNegativeLineNumber());
        //decl->setlineNumber(place->lineNumber());

        place->insertStmtBefore(*decl, *scope);
    }

    decl->setVariant(VAR_DECL_90);
    return decl;
}

//vars in 'symbolsToDeclare' may be various type 
//TODO add inits parameter
vector<SgStatement*> makeDeclaration(const vector<SgSymbol*>& symbolsToDeclare, SgStatement* where, vector<SgExpression*>* inits)
{
    vector<SgStatement*> allDecls;
    map<string, map<string, vector<SgSymbol*>>> groups;
    for (auto& toDec : symbolsToDeclare)
    {
        string newDecl = makeDeclaration(NULL, vector<SgSymbol*> { toDec })->unparse();
        auto it = newDecl.find("::");

        if (it == string::npos)
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

        groups[newDecl.substr(0, it)][toDec->identifier()].push_back(toDec);
    }

    for (auto& toDecls : groups)
    {
        vector<SgSymbol*> unitedList;
        for (auto& elem : toDecls.second)
            unitedList.push_back(elem.second[0]);

        allDecls.push_back(makeDeclaration(where, unitedList));
    }
    return allDecls;
}

int getNextFreeLabel()
{
    PTR_LABEL lab;
    set<int> used;
    for (lab = PROJ_FIRST_LABEL(); lab; lab = LABEL_NEXT(lab))
        used.insert(LABEL_STMTNO(lab));

    for (int z = 1; z < 99999; ++z)
        if (used.find(z) == used.end())
            return z;
    return -1;
}

static void recFillUsedVars(SgExpression *exp, map<string, SgSymbol*> &vars)
{
    if (exp)
    {
        if (exp->symbol() && (exp->variant() == VAR_REF || exp->variant() == ARRAY_REF || exp->variant() == CONST_REF)
            && !(exp->symbol()->attributes() & PRIVATE_BIT))
        {
            const auto key = exp->symbol()->identifier();
            if (vars.find(key) == vars.end())
                vars.insert(make_pair(key, exp->symbol()));
        }
        recFillUsedVars(exp->lhs(), vars);
        recFillUsedVars(exp->rhs(), vars);
    }
}

static inline void fillUsedVars(SgStatement *st, map<string, SgSymbol*> &vars)
{
    if (st)
    {
        for (int i = 0; i < 3; ++i)
        {
            SgExpression *exp = st->expr(i);
            recFillUsedVars(exp, vars);
        }
    }
}

static inline void joinMaps(map<string, SgSymbol*> &map1, const map<string, SgSymbol*> &map2, const map<string, SgSymbol*> &exept = map<string, SgSymbol*>())
{
    for (auto &pair : map2)
        if (exept.find(pair.first) == exept.end() && map1.find(pair.first) == map1.end())
            map1.insert(pair);
}

void fillVisibleInUseVariables(SgStatement *useSt, map<string, SgSymbol*> &vars)
{
    if (useSt)
    {
        if (useSt->variant() == USE_STMT)
        {
            bool only = false;

            map<string, SgSymbol*> localVars;  // local module variables
            map<string, SgSymbol*> useVars;    // variables from other modules
            map<string, SgSymbol*> renamedVas; // renamed variables
            map<string, SgSymbol*> originVars; // origin variables names in USE_STAT

            // check USE_STAT
            // fill from ONLY_NODE and RENAME_NODE
            SgExpression *ex = useSt->expr(0);
            if (ex && ex->variant() == ONLY_NODE)
            {
                only = true;
                ex = ex->lhs();
            }

            for (auto exI = ex; exI; exI = exI->rhs())
            {
                if (exI->lhs()->variant() == RENAME_NODE)
                {
                    SgExpression *ren = exI->lhs();
                    if (ren->lhs()->symbol() && ren->rhs() && ren->rhs()->symbol())
                    {
                        if (renamedVas.find(ren->lhs()->symbol()->identifier()) == renamedVas.end())
                            renamedVas.insert(make_pair(ren->lhs()->symbol()->identifier(), ren->lhs()->symbol()));
                        if (originVars.find(ren->rhs()->symbol()->identifier()) == originVars.end())
                            originVars.insert(make_pair(ren->rhs()->symbol()->identifier(), ren->rhs()->symbol()));
                    }
                    else if (only && ren->lhs()->symbol())
                    {
                        if (renamedVas.find(ren->lhs()->symbol()->identifier()) == renamedVas.end())
                            renamedVas.insert(make_pair(ren->lhs()->symbol()->identifier(), ren->lhs()->symbol()));
                    }
                }
            }

            if (!only)
            {
                // check module
                const string modName(useSt->symbol()->identifier());
                vector<SgStatement*> modules;
                findModulesInFile(useSt->getFile(), modules);

                bool found = false;
                for (auto i = 0; i < modules.size(); ++i)
                {
                    if (modName == modules[i]->symbol()->identifier())
                    {
                        found = true;

                        vector<SgStatement*> useStats;
                        for (SgStatement *st = modules[i]->lexNext(); st != modules[i]->lastNodeOfStmt(); st = st->lexNext())
                        {
                            if (st->variant() == USE_STMT)
                                useStats.push_back(st);
                            else if (st->variant() == CONTAINS_STMT)
                                break;
                            else
                                fillUsedVars(st, localVars);
                        }

                        for (auto &useSt : useStats)
                        {
                            map<string, SgSymbol*> visibleVars;
                            fillVisibleInUseVariables(useSt, visibleVars);
                            joinMaps(useVars, visibleVars);
                        }

                        break;
                    }
                }

                if (!found)
                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
            }

            // fill vars
            // TODO: check renaming
            joinMaps(vars, useVars, originVars);
            joinMaps(vars, localVars, originVars);
            joinMaps(vars, renamedVas);
        }
    }
}

Variable::Variable(SgFile* file, SgStatement* function, SgSymbol* symbol, const std::string& name, const varType type, const int position) :
                   symbol(symbol), name(name), type(type), position(position)
{
    declPace = declaratedInStmt(symbol);
    allUse.push_back(CommonVariableUse(file, function, symbol));
}

string preprocDataString(string data, bool full)
{
    string ret = "";


    for (int z = 0; z < data.size(); ++z)
    {
        if (data[z] == '\t')
            data[z] = ' ';
        else if (data[z] == '\r')
            data[z] = ' ';
    }

    auto it = data.find("DATA");
    if (it == string::npos)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    else
        it += 4;
    if (data[it] != ' ')
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    else
        it++;

    if (full)
        ret = "DATA ";

    int i = it;
    while (i != data.size())
    {
        if (data[i] != ' ')
            ret += data[i++];
        else
            i++;
    }
    return ret;
}

map<string, string> splitData(const set<SgValueExp*>& dataStats)
{
    map<string, string> result;
    for (auto& dataV : dataStats)
    {
        char* value = dataV->thellnd->entry.string_val;
        string dataS(value);
        convertToUpper(dataS);
        dataS = preprocDataString(dataS, false);

        convertToLower(dataS);
        string currS = "";
        string currV = "";
        int startV = -1;
        for (int z = 0; z < dataS.size(); ++z)
        {
            if (startV == -1)
            {
                if (dataS[z] == '/')
                    startV = 0;
                else if (dataS[z] != ' ' && dataS[z] != ',')
                    currS += dataS[z];
            }
            else if (startV == 0)
            {
                if (dataS[z] == '/')
                {
                    startV = -1;
                    result[currS] = currV;
                    currS = currV = "";
                }
                else if (dataS[z] != ' ')
                    currV += dataS[z];
            }            
        }
    }

    return result;
}

void extractComments(SgStatement* where, const string& what)
{
    if (BIF_CMNT(where->thebif) && CMNT_STRING(BIF_CMNT(where->thebif)))
    {
        char* str = CMNT_STRING(BIF_CMNT(where->thebif));
        string source(str);
        removeSubstrFromStr(source, what.c_str());
        sprintf(str, "%s", source.c_str());
    }
}

void getVariables(SgExpression* ex, set<string>& variables, const set<int> variants)
{
    if (ex)
    {
        if (variants.find(ex->variant()) != variants.end())
            variables.insert(OriginalSymbol(ex->symbol())->identifier());

        getVariables(ex->rhs(), variables, variants);
        getVariables(ex->lhs(), variables, variants);
    }
}

void getVariables(SgExpression* ex, set<SgSymbol*>& variables, const set<int> variants)
{
    if (ex)
    {
        if (variants.find(ex->variant()) != variants.end())
            variables.insert(OriginalSymbol(ex->symbol()));

        getVariables(ex->rhs(), variables, variants);
        getVariables(ex->lhs(), variables, variants);
    }
}

template<typename T>
set<T> getAllVariables(SgStatement* stFrom, SgStatement* stTo, const set<int>& variants)
{
    set<T> vars;
    for (auto stmt = stFrom; stmt != stTo; stmt = stmt->lexNext())
        for (int i = 0; i < 3; ++i)
            getVariables(stmt->expr(i), vars, variants);

    return vars;
}
template set<string>    getAllVariables(SgStatement* stFrom, SgStatement* stTo, const set<int>& variants);
template set<SgSymbol*> getAllVariables(SgStatement* stFrom, SgStatement* stTo, const set<int>& variants);

void shiftLines(SgProject* project, bool print)
{
    map<string, int> shifts;
    //shift lines if modules included
    const string shiftInfo = "!SPF NUM FILES";
    for (int z = 0; z < project->numberOfFiles(); ++z)
    {
        SgFile* file = &(project->file(z));
        const string fileN = file->filename();

        int shiftN = 0;
        for (auto st = file->firstStatement(); st; st = st->lexNext())
        {
            if (st->comments())
            {
                string comms = st->comments();
                auto it = comms.find(shiftInfo);
                if (it != string::npos)
                {
                    it += shiftInfo.size() + 1;

                    if (sscanf(comms.c_str() + it, "%d", &shiftN) == -1)
                    {
                        __spf_print(print, "comm %s, shifted %s\n", comms.c_str(), comms.c_str() + it);
                        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
                    }
                    break;
                }
            }
        }

        if (shiftN == 0)
            continue;
        shifts[fileN] = shiftN + 1; // with comment !SPF NUM FILES
        __spf_print(print, "  shift by %d for %s\n", shiftN, fileN.c_str());
    }

    for (int z = 0; z < project->numberOfFiles(); ++z)
    {
        SgFile* file = &(project->file(z));
        for (SgStatement* st = file->firstStatement()->lexNext(); st; st = st->lexNext())
        {
            string currF = st->fileName();
            auto it = shifts.find(currF);
            if (it != shifts.end() && file->filename() == currF)
                st->setlineNumber(st->lineNumber() - it->second);
        }
    }
}

SgProject* createProject(const char* proj_name, 
                         vector<ParallelRegion*>& parallelRegions,
                         vector<ParallelRegion*>& subs_parallelRegions,
                         map<string, vector<SgStatement*>>& hiddenData, 
                         map<string, int>& filesNameWithoutExt,
                         map<string, set<string>>& moduleUsesByFile, 
                         map<string, string>& moduleDecls,
                         map<string, map<SgStatement*, vector<SgStatement*>>>& exctactedModuleStats,
                         bool printSymbTable)
{
    Statement::setSapforRegime();
    Statement::deactiveConsistentchecker();
    sortFilesBySize(proj_name);
    SgProject* project = new SgProject(proj_name);
    addNumberOfFileToAttribute(project);

    if (printSymbTable) {
        for (int z = 0; z < project->numberOfFiles(); ++z) {
            SgFile* file = &(project->file(z));
            printf("===== TABLE FOR FILE %s\n", file->filename());
            printSymbolTable(file, "");
        }
        exit(0);
    }

    parallelRegions.push_back(new ParallelRegion(0, "DEFAULT"));
    subs_parallelRegions.push_back(new ParallelRegion(0, "DEFAULT"));

    Statement::activeConsistentchecker();
    Statement::activeDeprecatedchecker();

    //hide duplicated functions
    for (int z = 0; z < project->numberOfFiles(); ++z)
    {
        SgFile* file = &(project->file(z));
        //file->unparsestdout();

        const string fileN = file->filename();
        auto first = file->firstStatement();
        SgStatement* lastValid = NULL;

        const string toFind = "!SPF SHADOW FILES";
        for (SgStatement* st = first->lexNext(), *stPrev = first;
            st;
            st = st->lexNext(), stPrev = stPrev->lexNext())
        {
            if (st->comments())
            {
                string comm(st->comments());
                if (comm.find(toFind) != string::npos)
                {
                    if (st->variant() == CONTROL_END)
                    {
                        extractComments(st, toFind);
                        lastValid = st;
                    }
                    else
                        lastValid = stPrev;
                    break;
                }
            }
        }
        
        if (lastValid)
        {
            vector<SgStatement*> toExtract;
            auto st = lastValid->lexNext();
            while (st)
            {
                toExtract.push_back(st);
                st = st->lastNodeOfStmt();
                st = st->lexNext();
            }

            __spf_print(1, "hidden data for file '%s'\n", file->filename());
            set<int> validVars = { PROG_HEDR, FUNC_HEDR, PROC_HEDR, BLOCK_DATA, MODULE_STMT };
            for (auto& elem : toExtract)
            {
                int var = elem->variant();
                if (validVars.find(var) == validVars.end())
                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

                __spf_print(1, "  [%s, %d]\n", elem->fileName(), elem->lineNumber());
                hiddenData[fileN].push_back(elem->extractStmt());
            }
            lastValid->setLexNext((SgStatement*)NULL);
        }
    }

    //hide excluded modules
    const set<string> excludedMods = getExcludedModules();

    for (int z = 0; z < project->numberOfFiles(); ++z)
    {
        SgFile* file = &(project->file(z));

        const string fileN = file->filename();
        auto first = file->firstStatement();
        SgStatement* lastValid = NULL;

        vector<SgStatement*> toExtract;
        for (SgStatement* st = first->lexNext(); st; st = st->lexNext())
        {
            if (st->variant() == USE_STMT)
            {
                string name = st->symbol()->identifier();
                if (excludedMods.find(name) != excludedMods.end())
                {
                    SgStatement* before = st->lexNext();
                    do
                    {
                        if (before->variant() == USE_STMT)
                        {
                            if (excludedMods.find(before->symbol()->identifier()) != excludedMods.end())
                            {
                                before = before->lexNext();
                                continue;
                            }
                        }
                        break;
                    } while (true);

                    exctactedModuleStats[fileN][before].push_back(st);
                    toExtract.push_back(st);
                }
            }
        }

        for (auto& elem : toExtract)
            elem->extractStmt();
    }

    //dump symbol table and symbol use
    /*printSymbolTable(current_file);
    for (auto st = current_file->firstStatement()->lexNext(); st; st = st->lexNext())
    {
        if (st->variant() != MODULE_STMT && isSgProgHedrStmt(st) == NULL)
        {
            set<SgSymbol*> refs;
            for (int z = 0; z < 3; ++z)
                if (st->expr(z))
                    fillAllSymbs(st->expr(z), refs);

            if (refs.size())
            {
                printf("FOR STATEMENT ON LINE %d:\n", st->lineNumber());
                st->unparsestdout();
                printf("refs ids:");
                for (auto& elem : refs)
                    printf(" %d (%s)", elem->id(), elem->identifier());
                printf(";\n\n");
            }
        }
    }*/

    set<string> filesInProj;
    //preprocess module includes
    for (int z = 0; z < project->numberOfFiles(); ++z)
        filesInProj.insert((project->file(z)).filename());

    //check main unit
    findMainUnit(project, SPF_messages);

    set<string> globalFunctions;
    for (int z = 0; z < project->numberOfFiles(); ++z)
    {
        SgFile* file = &(project->file(z));

        string name = OnlyName(file->filename());
        if (filesNameWithoutExt.find(name) == filesNameWithoutExt.end())
            filesNameWithoutExt[name] = 1;
        else
            filesNameWithoutExt[name]++;

        fillModuleUse(file, moduleUsesByFile, moduleDecls);

        int funcCount = file->numberOfFunctions();
        for (int k = 0; k < funcCount; ++k)
        {
            auto func = file->functions(k);
            if (func->controlParent()->variant() == GLOBAL)
                globalFunctions.insert(func->symbol()->identifier());
        }
    }
    filterModuleUse(moduleUsesByFile, moduleDecls);

    //shift lines if modules included
    shiftLines(project);

    map<string, vector<Function>> functions;
    for (int z = 0; z < project->numberOfFiles(); ++z)
        fillFunctionInfo(&(project->file(z)), functions);

    set<string> needToResolve;
    for (auto& func : functions)
        if (func.second.size() > 1)
            needToResolve.insert(func.first);

    for (int z = 0; z < project->numberOfFiles(); ++z)
        resolveFunctionCalls(&(project->file(z)), needToResolve, functions);

    for (int z = 0; z < project->numberOfFiles(); ++z)
        correctModuleProcNames(&(project->file(z)), globalFunctions);


    for (int z = 0; z < project->numberOfFiles(); ++z)
        removeExecutableFromModuleDeclaration(&(project->file(z)), filesInProj, hiddenData[project->file(z).filename()]);

    for (int z = 0; z < project->numberOfFiles(); ++z)
    {
        SgFile* file = &(project->file(z));
        correctModuleSymbols(file);
        //file->unparsestdout();
    }

    if (detectMpiCalls(project, SPF_messages))
    {
        sharedMemoryParallelization = 1;
        keepDvmDirectives = 0;
        ignoreIO = 1;
        parallizeFreeLoops = 0;
    }

    return project;
}

static bool isIntrinsic(SgType* base)
{
    switch (base->variant())
    {
    case T_INT:
    case T_FLOAT:
    case T_BOOL:
    case T_DOUBLE:
    case T_COMPLEX:
    case T_DCOMPLEX:
    case T_CHAR:
        return true;
    default:
        break;
    }
    return false;
}

bool isArrayType(SgType* type)
{    
    if (type)
    {
        SgType* base = type->baseType();
        if (base && 
            type->variant() != T_STRING && 
            base->variant() != T_STRING &&
            (IS_INTRINSIC_TYPE(base) || isIntrinsic(base)))
            return true;
    }
    return false;
}

bool isArrayRef(SgExpression* ex)
{
    if (ex && ex->variant() == ARRAY_REF)
    {
        SgArrayRefExp* arrayRef = isSgArrayRefExp(ex);
        if (arrayRef)
            if (isArrayType(ex->symbol()->type()))
            {
                auto type = isSgArrayType(ex->symbol()->type());
                if (type && type->dimension())
                    return true;
            }
    }
    return false;
}

bool isStringArrayType(SgType* type)
{
    if (type)
    {
        SgType* base = type->baseType();
        if (base && (type->variant() == T_STRING || base->variant() == T_STRING))
            return true;
    }
    return false;
}

void moveLabelBefore(SgStatement* st, SgStatement* next)
{
    SgLabel* lab = next ? next->label() : st->label();
    if (lab)
    {
        SgStatement* cont = new SgContinueStmt();
        cont->setLabel(*lab);

        if (next)
            next->deleteLabel(true);
        else
            st->deleteLabel(true);

        if (next)
        {
            cont->setFileName(next->fileName());
            cont->setProject(next->getProject());
            cont->setFileId(next->getFileId());
            cont->setlineNumber(next->lineNumber());
        }
        else
        {
            cont->setFileName(st->fileName());
            cont->setProject(st->getProject());
            cont->setFileId(st->getFileId());
            cont->setlineNumber(getNextNegativeLineNumber());
        }
        st->insertStmtBefore(*cont, *st->controlParent());
    }
}

bool isEqSymbols(SgSymbol* sym1, SgSymbol* sym2)
{
    if (!sym1 || !sym2)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    return string(OriginalSymbol(sym1)->identifier()) == string(OriginalSymbol(sym2)->identifier());

    
}

set<string> getAllFilesInProject()
{
    set<string> res;
    const string currId = current_file->filename();
    for (int z = 0; z < CurrentProject->numberOfFiles(); ++z)
        res.insert(CurrentProject->file(z).filename());    
    if (SgFile::switchToFile(currId) == -1)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    return res;
}

void LogIftoIfThen(SgStatement* stmt)
{
    SgControlEndStmt* control = new SgControlEndStmt();
    stmt->setVariant(IF_NODE);
    stmt->lexNext()->insertStmtAfter(*control, *stmt);

    if (stmt->numberOfAttributes(OMP_MARK) > 0)
        control->addAttribute(OMP_MARK);
}

void removeSpecialCommentsFromProject(SgFile* file)
{
    //remove SAPFOR comment with version
    SgStatement* stF = file->firstStatement()->lexNext();
    while (stF)
    {
        if (stF && stF->comments())
        {
            string oldComment(stF->comments());
            auto it = oldComment.find("! *** generated by SAPFOR");
            if (it != string::npos)
            {
                auto endIt = oldComment.find("! *** generated by SAPFOR", it + 1);
                endIt = oldComment.find("\n", (endIt != string::npos) ? endIt : it);

                string newComment = oldComment.substr(0, it) + oldComment.substr(endIt + 1);
                stF->delComments();
                if (newComment != "")
                    stF->setComments(newComment.c_str());

                if (genVersionDone.find(file) != genVersionDone.end())
                    genVersionDone.erase(file);
            }
        }
        stF = stF->lexNext();
    } 
}

//TODO: for gen_block and other variations, and redistributions dirs
void getMaxMinBlockDistribution(SgFile* file, pair<int, int>& min_max)
{
    SgStatement* st = file->firstStatement();
    while (st)
    {
        if (isDVM_stat(st))
        {
            if (st->variant() == DVM_DISTRIBUTE_DIR || st->variant() == DVM_VAR_DECL)
            {
                for (int z = 0; z < 3; ++z)
                {
                    SgExpression* ex = st->expr(z);
                    queue<SgExpression*> q;

                    if (ex)
                    {
                        q.push(ex);
                        int blockCount = 0;
                        while (q.size())
                        {
                            ex = q.front();
                            q.pop();

                            if (ex->rhs())
                                q.push(ex->rhs());
                            if (ex->lhs())
                                q.push(ex->lhs());

                            if (ex->variant() == BLOCK_OP)
                                blockCount++;
                        }

                        if (blockCount)
                        {
                            if (min_max == make_pair(-1, -1))
                                min_max = make_pair(blockCount, blockCount);
                            else
                            {
                                min_max.first = std::min(min_max.first, blockCount);
                                min_max.second = std::max(min_max.second, blockCount);
                            }
                        }
                    }
                }
            }
        }
        st = st->lexNext();
    }
}

//TODO: need to add to includes
void addPrivatesToArraysFromGUI(SgFile* file, const map<tuple<int, string, string>, pair<DIST::Array*, DIST::ArrayAccessInfo*>>& declaredArrays,
                                const map<string, int>& distrStateFromGUI)
{
    map<SgStatement*, set<SgSymbol*>> added;

    for (auto& arrayPair : declaredArrays)
    {
        DIST::Array* array = arrayPair.second.first;
        string key = array->GetIndepUniqName();
        auto it = distrStateFromGUI.find(key);
        if (it == distrStateFromGUI.end())
            continue;

        if (it->second != DIST::NO_DISTR)
            continue;
        
        for (auto& decl : array->GetDeclInfoWithSymb())
        {
            auto& place = decl.first;
            const auto& symb = decl.second->GetOriginal();

            if (place.first != file->filename())
                continue;

            SgStatement* declSt = SgStatement::getStatementByFileAndLine(place.first, place.second);
            checkNull(declSt, convertFileName(__FILE__).c_str(), __LINE__);

            added[declSt].insert(symb);
        }
    }

    for (auto& toInsert : added)
    {
        vector<SgExpression*> list;
        for (auto& elem : toInsert.second)
            list.push_back(new SgVarRefExp(elem));
        
        SgStatement* op = new SgStatement(SPF_ANALYSIS_DIR);
        op->setExpression(0, new SgExpression(SPF_PROCESS_PRIVATE_OP, makeExprList(list)));

        toInsert.first->insertStmtBefore(*op, *toInsert.first->controlParent());
    }
}