finalyze moving

src/ParallelizationRegions/ParRegions.cpp

@@ -0,0 +1,896 @@
+#include "../Utils/leak_detector.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <vector>
+#include <map>
+#include <string>
+#include <algorithm>
+
+#include "dvm.h"
+#include "ParRegions.h"
+#include "../Utils/utils.h"
+#include "../Utils/SgUtils.h"
+#include "../GraphCall/graph_calls_func.h"
+#include "../GraphLoop/graph_loops.h"
+#include "../Distribution/Distribution.h"
+#include "../ExpressionTransform/expr_transform.h"
+
+using std::vector;
+using std::string;
+using std::pair;
+using std::make_pair;
+using std::map;
+using std::set;
+using std::wstring;
+
+extern void createMapLoopGraph(map<int, LoopGraph*> &sortedLoopGraph, const vector<LoopGraph*> *loopGraph);
+
+static map<string, int> regionIdByName;
+static map<string, ParallelRegion*> regionByName;
+
+static int regionIdConuter = 1;
+
+void clearRegionStaticData()
+{
+    regionIdByName.clear();
+    regionByName.clear();
+    regionIdConuter = 1;
+}
+
+
+static inline void extendRegionInfo(SgStatement *st, map<string, pair<Statement*, Statement*>> &startEnd, map<string, pair<int, int>> &lines_, bool addEndSt = false)
+{
+    auto it_st_en = startEnd.find(st->fileName());
+    if (it_st_en == startEnd.end())
+    {
+        startEnd.insert(it_st_en, make_pair(st->fileName(), make_pair<Statement*, Statement*>(new Statement(st), NULL)));
+        lines_[st->fileName()] = make_pair(st->lineNumber(), 0);
+    }
+    else
+    {
+        if (addEndSt)
+        {
+            Statement *toAdd = new Statement(st);
+            it_st_en->second.second = toAdd;
+        }
+        lines_[st->fileName()].second = st->lineNumber();
+    }
+}
+
+static void findFuncCalls(SgExpression *ex, set<string> &calls, const string &prefix, const map<string, FuncInfo*>& mapFuncs)
+{
+    if (ex == NULL)
+        return;
+
+    if (ex->variant() == FUNC_CALL)
+    {
+        string fullName = ex->symbol()->identifier();
+        //check contains
+        if (mapFuncs.find(prefix + fullName) != mapFuncs.end())
+            fullName = prefix + fullName;
+        calls.insert(fullName);
+    }
+
+    findFuncCalls(ex->lhs(), calls, prefix, mapFuncs);
+    findFuncCalls(ex->rhs(), calls, prefix, mapFuncs);
+}
+
+static inline SgStatement* getParentStat(SgStatement *st)
+{
+    if (!st)
+        return NULL;
+
+    SgStatement *iterator = st;
+
+    while (iterator->variant() != PROG_HEDR && iterator->variant() != PROC_HEDR && iterator->variant() != FUNC_HEDR)
+        iterator = iterator->controlParent();
+
+    while (iterator->controlParent()->variant() == PROG_HEDR || iterator->controlParent()->variant() == PROC_HEDR || iterator->controlParent()->variant() == FUNC_HEDR)
+        iterator = iterator->controlParent();
+
+    return iterator;
+}
+
+static void updateRegionInfo(SgStatement *st, map<string, pair<Statement*, Statement*>> &startEnd, map<string, pair<int, int>> &lines_,
+                             set<string> &funcCallFromReg, const map<string, FuncInfo*> &mapFuncs)
+{    
+    string containsPrefix = "";
+    SgStatement *st_ps = getParentStat(st);
+
+    if (st_ps->variant() == PROC_HEDR || st_ps->variant() == PROG_HEDR || st_ps->variant() == FUNC_HEDR)
+        containsPrefix = st_ps->symbol()->identifier() + string(".");
+
+    extendRegionInfo(st, startEnd, lines_);
+    if (st->variant() == PROC_STAT)
+    {
+        string fullName = st->symbol()->identifier();
+        //check contains
+        if (mapFuncs.find(containsPrefix + fullName) != mapFuncs.end())
+            fullName = containsPrefix + fullName;
+        funcCallFromReg.insert(fullName);
+    }
+
+    for (int z = 0; z < 3; ++z)
+        findFuncCalls(st->expr(z), funcCallFromReg, containsPrefix, mapFuncs);
+}
+
+static void fillArrayNamesInReg(set<string> &usedArrayInRegion, SgExpression *exp)
+{
+    if (exp)
+    {
+        if (exp->variant() == ARRAY_REF && OriginalSymbol(exp->symbol()))
+            usedArrayInRegion.insert(OriginalSymbol(exp->symbol())->identifier());
+
+        fillArrayNamesInReg(usedArrayInRegion, exp->lhs());
+        fillArrayNamesInReg(usedArrayInRegion, exp->rhs());
+    }
+}
+
+static bool filterUserDirectives(set<string> &usedArrayInRegion, vector<Statement*> &userDvmDirs, vector<Statement*> &userDvmDirsF)
+{
+    vector<Statement*> filtered;
+    bool changed = false;
+    for (auto &distr : userDvmDirs)
+    {
+        set<string> usedIndir;
+        for (int i = 0; i < 3; ++i)
+            fillArrayNamesInReg(usedIndir, distr->GetOriginal()->expr(i));
+
+        bool ok = false;
+        for (auto &elem : usedIndir)
+        {
+            if (usedArrayInRegion.find(elem) != usedArrayInRegion.end())
+            {
+                ok = true;
+                break;
+            }
+        }
+
+        if (ok)
+        {
+            usedArrayInRegion.insert(usedIndir.begin(), usedIndir.end());
+            userDvmDirsF.push_back(distr);
+            changed = true;
+        }
+        else
+            filtered.push_back(distr);
+    }
+    userDvmDirs = filtered;
+
+    return changed;
+}
+
+static void filterUserDirectives(ParallelRegion *currReg, set<string> usedArrayInRegion, vector<Statement*> userDvmDistrDirs,
+                                 vector<Statement*> userDvmAlignDirs, vector<Statement*> userDvmShadowDirs)
+{
+    vector<Statement*> userDvmDistrDirsF;
+    vector<Statement*> userDvmAlignDirsF;
+    vector<Statement*> userDvmShadowDirsF;
+
+    bool changed = false;
+    do
+    {
+        changed = false;
+
+        bool ret = filterUserDirectives(usedArrayInRegion, userDvmDistrDirs, userDvmDistrDirsF);
+        changed = changed || ret;
+
+        ret = filterUserDirectives(usedArrayInRegion, userDvmAlignDirs, userDvmAlignDirsF);
+        changed = changed || ret;
+
+        ret = filterUserDirectives(usedArrayInRegion, userDvmShadowDirs, userDvmShadowDirsF);
+        changed = changed || ret;
+                
+    } while (changed);
+
+    currReg->AddUserDirectives(userDvmDistrDirsF, DVM_DISTRIBUTE_DIR);
+    currReg->AddUserDirectives(userDvmAlignDirsF, DVM_ALIGN_DIR);
+    currReg->AddUserDirectives(userDvmShadowDirsF, DVM_SHADOW_DIR);
+}
+
+static void setExplicitFlag(const string &name, const map<string, FuncInfo*> &mapFuncs)
+{
+    auto it = mapFuncs.find(name);
+    if (it != mapFuncs.end())
+        it->second->inRegion = 1;
+}
+
+static void fillDvmDirs(SgStatement *st, vector<Statement*> &userDvmDistrDirs, vector<Statement*> &userDvmAlignDirs,
+                        vector<Statement*> &userDvmShadowDirs, vector<Statement*> &userDvmRealignDirs, vector<Statement*> &userDvmRedistrDirs,
+                        vector<Statement*>& userDvmRealignDirsDef, vector<Statement*>& userDvmRedistrDirsDef,
+                        const bool &regionStarted, const map<int, LoopGraph*> &allLoopsInFile)
+{
+    switch (st->variant())
+    {
+    case DVM_VAR_DECL:
+    {
+        string unparsed = st->unparse();
+        convertToLower(unparsed);
+        if (unparsed.find("distribute") != string::npos)
+            userDvmDistrDirs.push_back(new Statement(st));
+        else if (unparsed.find("align") != string::npos)
+            userDvmAlignDirs.push_back(new Statement(st));
+    }
+    break;
+    case DVM_DISTRIBUTE_DIR:
+        userDvmDistrDirs.push_back(new Statement(st));
+        break;
+    case DVM_ALIGN_DIR:
+        userDvmAlignDirs.push_back(new Statement(st));
+        break;
+    case DVM_SHADOW_DIR:
+        userDvmShadowDirs.push_back(new Statement(st));
+        break;
+    case DVM_REALIGN_DIR:
+        if (regionStarted)
+            userDvmRealignDirs.push_back(new Statement(st));
+        userDvmRealignDirsDef.push_back(new Statement(st));
+        break;
+    case DVM_REDISTRIBUTE_DIR:
+        //skip 22.04.2020
+        /*if (regionStarted)
+            userDvmRedistrDirs.push_back(new Statement(st));
+        userDvmRedistrDirsDef.push_back(new Statement(st));*/
+        break;
+    case DVM_PARALLEL_ON_DIR:
+        if (st->lexNext()->variant() == FOR_NODE)
+        {
+            SgForStmt* forStat = (SgForStmt*)(st->lexNext());
+            auto it = allLoopsInFile.find(forStat->lineNumber());
+            if (it != allLoopsInFile.end())
+                it->second->userDvmDirective = new Statement(st);
+        }
+        break;
+    default:
+        break;
+    }
+}
+
+static void checkForEmpty(SgStatement *start, SgStatement *end, vector<Messages>& messagesForFile)
+{
+    bool wasStarted = false;
+    int lineStarted = -1;
+
+    while (start != end)
+    {
+        if (start->variant() == CONTAINS_STMT)
+            break;
+
+        if (start->variant() == SPF_PARALLEL_REG_DIR)
+        {
+            wasStarted = true;
+            lineStarted = start->lineNumber();
+        }
+        else if (start->variant() == SPF_END_PARALLEL_REG_DIR)
+        {
+            if (wasStarted)
+            {
+                wstring messageE, messageR;
+                __spf_printToLongBuf(messageE, L"Empty parallel regions is forbidden.");
+                __spf_printToLongBuf(messageR, R151);
+                messagesForFile.push_back(Messages(ERROR, lineStarted, messageR, messageE, 3021));
+                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+            }
+        }
+        else
+        {
+            wasStarted = false;
+            lineStarted = -1;
+        }
+        start = start->lexNext();
+    }
+}
+
+void fillRegionLines(SgFile *file, vector<ParallelRegion*> &regions, vector<Messages>& messagesForFile, vector<LoopGraph*> *loops, vector<FuncInfo*> *funcs)
+{
+    map<string, FuncInfo*> mapFuncs;
+    if (funcs)
+        createMapOfFunc(*funcs, mapFuncs);
+
+    //fill default
+    ParallelRegion* defaultR = regions[0];
+    pair<int, int> lines;
+    //pair<SgStatement*, SgStatement*> startEndDef(NULL, NULL);
+    {
+        SgStatement* st = file->firstStatement();
+        st = st->lexNext();
+        lines = make_pair(st->lineNumber(), 0);
+        //startEndDef.first = st;
+
+        while (st)
+        {
+            st = st->lastNodeOfStmt();
+            lines.second = st->lineNumber();
+            //startEndDef.second = st;
+            st = st->lexNext();
+        }
+    }
+    defaultR->AddLines(lines, file->filename());
+
+    map<int, LoopGraph*> allLoopsInFile;
+    if (loops)
+        createMapLoopGraph(allLoopsInFile, loops);
+
+    //fill user's
+    int funcNum = file->numberOfFunctions();
+    string regionName = "";
+    map<string, pair<Statement*, Statement*>> startEnd;
+    map<string, pair<int, int>> lines_;
+    set<string> funcCallFromReg;
+    bool regionStarted = false;
+
+    vector<SgStatement*> toDel;
+    for (int i = 0; i < funcNum; ++i)
+    {
+        vector<Statement*> userDvmDistrDirs;
+        vector<Statement*> userDvmAlignDirs;
+        vector<Statement*> userDvmShadowDirs;
+        vector<Statement*> userDvmRealignDirs;
+        vector<Statement*> userDvmRedistrDirs;
+
+        vector<Statement*> userDvmRealignDirsDef;
+        vector<Statement*> userDvmRedistrDirsDef;
+
+        set<string> usedArrayInRegion;
+
+        SgStatement *st = file->functions(i);
+        SgStatement *lastNode = st->lastNodeOfStmt();
+
+        string containsPrefix = "";
+        SgStatement *st_cp = st->controlParent();
+        if (st_cp->variant() == PROC_HEDR || st_cp->variant() == PROG_HEDR || st_cp->variant() == FUNC_HEDR)
+            containsPrefix = st_cp->symbol()->identifier() + string(".");
+        const string funcName = containsPrefix + file->functions(i)->symbol()->identifier();
+
+        checkForEmpty(st, lastNode, messagesForFile);
+        while (st != NULL && st != lastNode)
+        {
+            if (st->variant() == CONTAINS_STMT)
+                break;
+
+            SgStatement *prev = st->lexPrev();
+            SgStatement *next = st->lexNext();
+
+            if (prev && prev->variant() == SPF_PARALLEL_REG_DIR)
+            {
+                SgStatement *data = prev;
+
+                regionStarted = true;
+                regionName = data->symbol()->identifier();
+                if (funcs)
+                {
+                    auto itFunc = mapFuncs.find(funcName);
+                    if (itFunc != mapFuncs.end())
+                    {
+                        itFunc->second->inRegion = 3;
+                        itFunc->second->callRegions.insert(0);
+                    }
+                }
+            }
+
+            if (next && next->variant() == SPF_END_PARALLEL_REG_DIR)
+            {
+                updateRegionInfo(st, startEnd, lines_, funcCallFromReg, mapFuncs);
+                SgStatement *data = next;
+
+                lines.second = data->lineNumber();
+                regionStarted = false;
+
+                if (regionName == "")
+                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+                auto it = regionIdByName.find(regionName);
+                ParallelRegion *currReg = NULL;
+                if (it == regionIdByName.end())
+                {
+                    it = regionIdByName.insert(it, make_pair(regionName, regionIdConuter));
+                    currReg = new ParallelRegion(regionIdConuter, regionName);
+                    regions.push_back(currReg);
+                    regionByName[regionName] = currReg;
+                    regionName = "";
+                    regionIdConuter++;
+                }
+                else
+                    currReg = regionByName[regionName];
+
+                if (currReg == NULL)
+                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+                extendRegionInfo(st, startEnd, lines_, true);
+                for (auto itRegInfo = startEnd.begin(); itRegInfo != startEnd.end(); ++itRegInfo)
+                    currReg->AddLines(lines_[itRegInfo->first], itRegInfo->first, &itRegInfo->second);
+
+                for (auto &func : funcCallFromReg)
+                    currReg->AddFuncCalls(func);
+
+                filterUserDirectives(currReg, usedArrayInRegion, userDvmRedistrDirs, userDvmRealignDirs, userDvmShadowDirs);
+                currReg->AddUserDirectives(userDvmRealignDirs, DVM_REALIGN_DIR);
+                currReg->AddUserDirectives(userDvmRedistrDirs, DVM_REDISTRIBUTE_DIR);
+
+                startEnd.clear();
+                lines_.clear();
+                funcCallFromReg.clear();
+
+                userDvmRealignDirs.clear();
+                userDvmRedistrDirs.clear();
+
+                usedArrayInRegion.clear();
+            }
+
+
+            if (regionStarted)
+            {
+                updateRegionInfo(st, startEnd, lines_, funcCallFromReg, mapFuncs);
+                for (int i = 0; i < 3; ++i)
+                    fillArrayNamesInReg(usedArrayInRegion, st->expr(i));
+            }
+
+            fillDvmDirs(st, userDvmDistrDirs, userDvmAlignDirs, userDvmShadowDirs, userDvmRealignDirs, userDvmRealignDirs, userDvmRealignDirsDef, userDvmRealignDirsDef, regionStarted, allLoopsInFile);
+            if (st->variant() == DVM_SHADOW_GROUP_DIR || st->variant() == DVM_SHADOW_START_DIR || st->variant() == DVM_SHADOW_WAIT_DIR)
+                toDel.push_back(st);
+            st = st->lexNext();
+        }
+
+        //for default
+        defaultR->AddUserDirectives(userDvmDistrDirs, DVM_DISTRIBUTE_DIR);
+        defaultR->AddUserDirectives(userDvmAlignDirs, DVM_ALIGN_DIR);
+        defaultR->AddUserDirectives(userDvmShadowDirs, DVM_SHADOW_DIR);
+        defaultR->AddUserDirectives(userDvmRealignDirsDef, DVM_REALIGN_DIR);
+        defaultR->AddUserDirectives(userDvmRedistrDirsDef, DVM_REDISTRIBUTE_DIR);
+    }
+
+    vector<SgStatement*> modules;
+    findModulesInFile(file, modules);
+    regionStarted = false;
+
+    for (auto& mod : modules)
+    {
+        vector<Statement*> userDvmDistrDirs;
+        vector<Statement*> userDvmAlignDirs;
+        vector<Statement*> userDvmShadowDirs;
+        vector<Statement*> userDvmRealignDirs;
+        vector<Statement*> userDvmRedistrDirs;
+
+        vector<Statement*> userDvmRealignDirsDef;
+        vector<Statement*> userDvmRedistrDirsDef;
+
+
+        for (SgStatement* st = mod->lexNext(); st; st = st->lexNext())
+        {
+            if (isSgExecutableStatement(st))
+                break;
+            fillDvmDirs(st, userDvmDistrDirs, userDvmAlignDirs, userDvmShadowDirs, userDvmRealignDirs, userDvmRealignDirs, userDvmRealignDirsDef, userDvmRealignDirsDef, regionStarted, allLoopsInFile);
+            if (st->variant() == DVM_SHADOW_GROUP_DIR)
+                toDel.push_back(st);
+        }
+
+        //for default
+        defaultR->AddUserDirectives(userDvmDistrDirs, DVM_DISTRIBUTE_DIR);
+        defaultR->AddUserDirectives(userDvmAlignDirs, DVM_ALIGN_DIR);
+        defaultR->AddUserDirectives(userDvmShadowDirs, DVM_SHADOW_DIR);
+        defaultR->AddUserDirectives(userDvmRealignDirsDef, DVM_REALIGN_DIR);
+        defaultR->AddUserDirectives(userDvmRedistrDirsDef, DVM_REDISTRIBUTE_DIR);
+    }
+
+    for (auto& elem : toDel)
+        elem->deleteStmt();
+}
+
+static void getAllLoops(vector<LoopGraph*> &loopGraph, vector<LoopGraph*> &loops)
+{
+    for (auto &elem : loopGraph)
+        loops.push_back(elem);
+
+    for (auto &elem : loopGraph)
+        getAllLoops(elem->children, loops);
+}
+
+void fillRegionLinesStep2(vector<ParallelRegion*> &regions, const map<string, vector<FuncInfo*>> &allFuncInfo, map<string, vector<LoopGraph*>> *loopGraph)
+{
+    map<string, FuncInfo*> funcMap;
+    createMapOfFunc(allFuncInfo, funcMap);
+
+    for (int i = 0; i < regions.size(); ++i)
+    {
+        if (regions[i]->GetName() != "DEFAULT")
+            for (auto &func : regions[i]->GetFuncCalls())
+                setExplicitFlag(func, funcMap);        
+    }
+
+    for (int i = 0; i < regions.size(); ++i)
+    {
+        if (regions[i]->GetName() != "DEFAULT")
+        {
+            set<string> uniqFuncCalls;
+            for (auto &elem : regions[i]->GetFuncCalls())
+                uniqFuncCalls.insert(elem);
+
+            bool wasChanged = 1;
+            while (wasChanged)
+            {
+                wasChanged = 0;
+                for (auto &uniqF : uniqFuncCalls)
+                {
+                    auto func = funcMap.find(uniqF);
+                    if (func != funcMap.end())
+                    {
+                        for (auto &calls : func->second->callsFrom)
+                        {
+                            auto it = uniqFuncCalls.find(calls);
+                            if (it == uniqFuncCalls.end())
+                            {
+                                uniqFuncCalls.insert(it, calls);
+                                wasChanged = 1;
+                            }
+                        }
+                    }
+                }
+            }
+
+            string toPrint = "";
+            for (auto &elem : uniqFuncCalls)
+            {
+                auto it = funcMap.find(elem);
+                if (it != funcMap.end())
+                {
+                    regions[i]->AddLines(it->second->linesNum, it->second->fileName);
+                    regions[i]->AddFuncCallsToAllCalls(it->second);
+
+                    if (it->second->inRegion == 0)
+                        it->second->inRegion = 2;
+
+                    it->second->callRegions.insert(i);
+
+                    toPrint += elem + " ";
+                }
+            }
+
+            if (toPrint != "")
+                __spf_print(1, "[%s]: funcs: %s\n", regions[i]->GetName().c_str(), toPrint.c_str());
+        }
+    }
+
+    // remove default region if user's found
+    if (regions.size() > 1)
+    {
+#ifdef _WIN32
+        removeFromCollection(regions[0]);
+#endif
+        delete regions[0];
+        regions.erase(regions.begin());
+    }
+
+    bool changed = true;
+    while (changed)
+    {
+        changed = false;
+        for (auto &func : funcMap)
+        {
+            if (func.second->inRegion != 0)
+                continue;
+
+            for (auto &callsFrom : func.second->callsFrom)
+            {
+                auto it = funcMap.find(callsFrom);
+                if (it != funcMap.end())
+                {
+                    if (it->second->inRegion > 0)
+                    {
+                        func.second->inRegion = 3;
+                        func.second->callRegions.insert(0);
+                        changed = true;
+                        break;
+                    }
+                    else
+                        func.second->callRegions.insert(0);
+                }
+            }
+        }
+    }
+
+    if (loopGraph)
+    {
+        //fill regions for loop 
+        vector<LoopGraph*> loops;
+        for (auto loop : *loopGraph)
+            getAllLoops(loop.second, loops);
+
+        for (auto &loop : loops)
+        {
+            const int currLine = loop->lineNum < -1 ? loop->loop->localLineNumber() : loop->lineNum;
+            set<ParallelRegion*> allRegs = getAllRegionsByLine(regions, loop->fileName, currLine);
+            if (allRegs.size() == 1)
+                loop->region = *(allRegs.begin());
+        }
+    }
+}
+
+int printParalleRegions(const char *fileName, vector<ParallelRegion*> &regions)
+{
+    FILE *file = fopen(fileName, "w");
+    if (file == NULL)
+    {
+        __spf_print(1, "can not open file %s\n", fileName);
+        return -1;
+    }
+
+    int num = 0;
+    for (auto &elem : regions)
+    {
+        fprintf(file, "*** #%d Parallel Region\n", num);
+        elem->print(file);
+        fprintf(file, "\n");
+        ++num;
+    }
+
+    fclose(file);
+    return 0;
+}
+
+static int getIntVal(SgExpression *ex)
+{
+    SgExpression* calc = CalculateInteger(ex->copyPtr());
+    if (calc->variant() == INT_VAL)
+        return calc->valueInteger();
+    else
+        return 0;
+}
+
+static void fillMultOp(SgExpression *ex, pair<string, pair<int, int>> &retVal)
+{
+    const int var = ex->lhs()->variant();
+    if (var == INT_VAL || var == CONST_REF || var == MINUS_OP) // -+a * X
+    {
+        retVal.first = ex->rhs()->symbol()->identifier();        
+        retVal.second.first = getIntVal(ex->lhs());        
+    }
+    else if (var == VAR_REF) // X * -+a
+    {
+        retVal.first = ex->lhs()->symbol()->identifier();
+        retVal.second.first = getIntVal(ex->rhs());
+    }
+}
+
+static pair<string, pair<int, int>> parseExpression(SgExpression *ex)
+{
+    pair<string, pair<int, int>> retVal;
+
+    if (ex)
+    {
+        if (ex->variant() == VAR_REF)
+        {
+            retVal.first = ex->symbol()->identifier();
+            retVal.second = make_pair(1, 0);
+        }
+        else if (ex->variant() == MULT_OP) // a * X
+            fillMultOp(ex, retVal);
+        else if (ex->variant() == ADD_OP || ex->variant() == SUBT_OP) // a * X +- b
+        {
+            int minus = (ex->variant() == ADD_OP) ? 1 : -1;
+            if (ex->lhs()->variant() == MULT_OP) // a * X +- b
+            {
+                fillMultOp(ex->lhs(), retVal);
+                retVal.second.second = getIntVal(ex->rhs()) * minus;
+            }
+            else if (ex->lhs()->variant() == INT_VAL) // b +- [a *] X
+            {
+                if (ex->rhs()->variant() == MULT_OP)
+                    fillMultOp(ex->rhs(), retVal);
+                else
+                {
+                    retVal.first = ex->rhs()->symbol()->identifier();
+                    retVal.second.first = minus;
+                }
+                retVal.second.second = getIntVal(ex->lhs()) * minus;
+            }
+            else if (ex->lhs()->variant() == VAR_REF && ex->rhs()->variant()) // X +- b
+            {
+                retVal.first = ex->lhs()->symbol()->identifier();
+                retVal.second.first = 1;
+                retVal.second.second = getIntVal(ex->rhs()) * minus;
+            }
+            else
+                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+        }
+        else if (ex->variant() == KEYWORD_VAL) // *
+        {
+            SgKeywordValExp* keyVal = (SgKeywordValExp*)ex;
+            if (keyVal->value())
+                retVal = make_pair(keyVal->value(), make_pair(0, 0));
+        }
+        else if (ex->variant() == DDOT || ex->variant() == INT_VAL || ex->variant() == CONST_REF) // ':' | 'digit' equal '*'
+            retVal = make_pair("*", make_pair(0, 0));
+        else
+        {
+            __spf_print(1, "align pattern: '%s'\n", ex->unparse());
+            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+        }
+    }
+    return retVal;
+}
+
+static void fillTemplate(SgExpression *ex, vector<pair<string, pair<int, int>>> &templateArray)
+{
+    for (SgExpression *list = ex; list; list = list->rhs())
+        templateArray.push_back(parseExpression(list->lhs()));
+}
+
+static int getTemplateDemention(const string &val, const vector<pair<string, pair<int, int>>> &alignWithTemplate)
+{
+    int ret = -1;
+    for (int k = 0; k < alignWithTemplate.size(); ++k)
+    {
+        if (alignWithTemplate[k].first == val)
+        {
+            ret = k;
+            break;
+        }
+    }
+    return ret;
+}
+
+static bool needToSkip(SgStatement* dir, vector<FuncInfo*> funcs)
+{
+    bool retVal = false;
+
+    auto iterator = dir;
+
+    while (iterator && iterator->variant() != PROG_HEDR && iterator->variant() != PROC_HEDR && iterator->variant() != FUNC_HEDR)
+        iterator = iterator->controlParent();
+    
+    if (iterator == NULL)
+        return retVal;
+
+    for (auto& elem : funcs)
+    {
+        if (elem->funcPointer->GetOriginal()->thebif == iterator->thebif)
+        {
+            if (elem->doNotAnalyze)
+                return true;
+            break;
+        }
+    }
+    return retVal;
+}
+
+bool buildGraphFromUserDirectives(const vector<Statement*> &userDvmAlignDirs, DIST::GraphCSR<int, double, attrType> &G, 
+                                  DIST::Arrays<int> &allArrays, const map<DIST::Array*, set<DIST::Array*>> &arrayLinksByFuncCalls, 
+                                  const set<DIST::Array*>& alignedArrays, set<DIST::Array*>& addedArrays,
+                                  const map<string, vector<FuncInfo*>>& funcsByFile)
+{
+    if (userDvmAlignDirs.size())
+    {
+        bool cleaned = false;
+
+        int t = 0;
+        for (auto &dir : userDvmAlignDirs)
+        {
+            const string file = dir->GetFileNameIncludedTo();
+            if (SgFile::switchToFile(file) == -1)
+                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+            auto it = funcsByFile.find(file);
+            
+            if (needToSkip(dir->GetOriginal(), it == funcsByFile.end() ? vector<FuncInfo*>() : it->second));
+                continue;
+
+            vector<DIST::Array*> alignArrays;
+            
+            vector<pair<string, pair<int, int>>> alignTemplate;
+
+            DIST::Array* alignWithArray = NULL;
+            vector<pair<string, pair<int, int>>> alignWithTemplate;
+
+            if (dir->expr(0) == NULL)
+                return true;
+
+            for (SgExpression *ex = dir->expr(0); ex; ex = ex->rhs())
+            {
+                SgExpression *val = ex->lhs();
+                if (val->variant() == ARRAY_REF && val->symbol())
+                {
+                    DIST::Array *newValue = getArrayFromDeclarated(declaratedInStmt(val->symbol()), val->symbol()->identifier());
+                    if (newValue)
+                        alignArrays.push_back(newValue);
+                    else
+                        return true;
+                }
+            }
+
+            map<DIST::Array*, set<DIST::Array*>> realAlignArrayRefs;
+            for (auto &access : alignArrays)
+                getRealArrayRefs(access, access, realAlignArrayRefs[access], arrayLinksByFuncCalls);
+
+            set<DIST::Array*> realAlignArrayRefsSet;
+            for (auto &elem : realAlignArrayRefs)
+                realAlignArrayRefsSet.insert(elem.second.begin(), elem.second.end());
+
+            if (dir->expr(1))
+                fillTemplate(dir->expr(1), alignTemplate);
+            else if (dir->variant() == DVM_VAR_DECL)
+                fillTemplate(dir->expr(2)->lhs()->lhs(), alignTemplate);
+            else
+                return true;
+
+            if (dir->expr(2))
+            {
+                SgExpression *ex = dir->expr(2);
+                if (ex->variant() == EXPR_LIST && ex->lhs()->variant() == ALIGN_OP)
+                    ex = ex->lhs()->rhs();
+
+                if (ex && ex->variant() == ARRAY_REF && ex->symbol())
+                {
+                    alignWithArray = getArrayFromDeclarated(declaratedInStmt(ex->symbol()), ex->symbol()->identifier());
+                    if (alignWithArray == NULL)
+                        return true;
+                    fillTemplate(ex->lhs(), alignWithTemplate);
+                }
+                else
+                    return true;
+            }
+            else
+                return true;
+
+            string tmp;            
+            for (auto& elem : realAlignArrayRefsSet)
+                tmp += elem->GetName() + " ";
+            __spf_print(1, "  align array%s '%s' from user dir in line %d\n", (realAlignArrayRefsSet.size() == 1 ? "" : "s"), tmp.c_str(), dir->lineNumber());
+            __spf_print(1, "  template align:\n");
+            for (int i = 0; i < alignTemplate.size(); ++i)
+                __spf_print(1, "    -- %d: %s -- [%d, %d]\n", i, alignTemplate[i].first.c_str(), alignTemplate[i].second.first, alignTemplate[i].second.second);
+            __spf_print(1, "  template align with:\n");
+            for (int i = 0; i < alignWithTemplate.size(); ++i)
+                __spf_print(1, "    -- %d: %s -- [%d, %d]\n", i, alignWithTemplate[i].first.c_str(), alignWithTemplate[i].second.first, alignWithTemplate[i].second.second);
+
+            for (int i = 0; i < alignTemplate.size(); ++i)
+            {
+                if (alignTemplate[i].first != "*")
+                {
+                    int dimT = getTemplateDemention(alignTemplate[i].first, alignWithTemplate);
+                    if (dimT == -1)
+                        continue;
+                    
+                    for (auto& array : realAlignArrayRefsSet)
+                    {
+                        if (!array->IsNotDistribute())
+                        {
+                            if (alignWithArray->IsTemplate() && alignWithArray->GetShortName().find("_r") != string::npos)
+                                continue;
+
+                            //TODO:
+                            /*if (alignedArrays.find(array) != alignedArrays.end())
+                                continue;*/
+                            addedArrays.insert(array);
+
+                            if (!cleaned)
+                            {
+                                cleaned = true;
+                                G.ClearGraphCSR();
+                            }
+
+                            DIST::AddArrayAccess(G, allArrays, array, alignWithArray, make_pair(i, dimT), 1.0, make_pair(make_pair(1, 0), alignWithTemplate[dimT].second), WW_link);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    return false;
+}
+
+void calculateLinesOfCode(vector<ParallelRegion*> &allRegions)
+{
+    for (auto &elem : allRegions)
+    {
+        auto allLines = elem->GetAllLines();
+        int lineCounter = 0;
+        for (auto &line : allLines)
+            for (auto &lineV : line.second)
+                lineCounter += (lineV.lines.second - lineV.lines.first);
+
+        __spf_print(1, "  Count of lines in region '%s' = %d\n", elem->GetName().c_str(), lineCounter);
+    }
+}

src/ParallelizationRegions/ParRegions.h

@@ -0,0 +1,402 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <algorithm>
+#include <set>
+#include <map>
+
+#include "../Distribution/DvmhDirective.h"
+#include "../Distribution/GraphCSR.h"
+#include "../Distribution/Distribution.h"
+#include "../Utils/AstWrapper.h"
+
+#if __SPF
+#include "../Utils/SgUtils.h"
+#endif
+
+struct ParallelRegionLines
+{
+    ParallelRegionLines() 
+    {
+        lines = std::make_pair(-1, -1);
+        stats = std::make_pair<Statement*, Statement*>(NULL, NULL);
+        intervalBefore = std::make_pair<Statement*, Statement*>(NULL, NULL);
+        intervalAfter = std::make_pair<Statement*, Statement*>(NULL, NULL);
+    }
+
+    ParallelRegionLines(const std::pair<int, int> &lines) : lines(lines) 
+    {
+        stats = std::make_pair<Statement*, Statement*>(NULL, NULL);
+        intervalBefore = std::make_pair<Statement*, Statement*>(NULL, NULL);
+        intervalAfter = std::make_pair<Statement*, Statement*>(NULL, NULL);
+    }
+
+    ParallelRegionLines(const std::pair<int, int> &lines, const std::pair<Statement*, Statement*> stats) : lines(lines), stats(stats)
+    {
+        intervalBefore = std::make_pair<Statement*, Statement*>(NULL, NULL);
+        intervalAfter = std::make_pair<Statement*, Statement*>(NULL, NULL);
+    }
+
+    bool operator==(const ParallelRegionLines &regionLines) const { return lines == regionLines.lines && stats == regionLines.stats; }
+    bool operator<(const ParallelRegionLines &otherLines) const { return lines.first < otherLines.lines.first; }
+
+    void print(FILE *fileOut)
+    {
+        fprintf(fileOut, " [%d -- %d]", lines.first, lines.second);
+        if (stats.first && stats.second)
+            fprintf(fileOut, " explicit\n");
+        else
+            fprintf(fileOut, "\n");
+    }
+
+    bool isImplicit() const { return stats.first == NULL || stats.second == NULL; }
+
+    // <start, end> lines
+    std::pair<int, int> lines;
+    // <start, end> stats
+    std::pair<Statement*, Statement*> stats;
+
+    // <start, end> interval
+    std::pair<Statement*, Statement*> intervalBefore;
+    std::pair<Statement*, Statement*> intervalAfter;
+};
+
+#if __SPF
+struct ParallelRegionArray
+{
+private:
+    std::string name;
+    std::string fileName;
+    SgSymbol *origSymbol;
+    SgSymbol *copySymbol;
+    std::vector<SgStatement*> declStatements;
+    std::vector<ParallelRegionLines> allLines;
+public:
+    explicit ParallelRegionArray(const std::string &name, const std::string &fileName, SgSymbol *origSymbol, SgSymbol *copySymbol,
+                                 const ParallelRegionLines &lines, std::vector<SgStatement*> &declStatements) :
+                                 name(name), fileName(fileName), origSymbol(origSymbol), copySymbol(copySymbol), declStatements(declStatements)
+    {
+        allLines.push_back(lines);
+    }
+
+    const std::string& getName() const { return name; }
+    const std::string& getFileName() const { return fileName; }
+    SgSymbol* getOrigSymbol() const { return origSymbol; }
+    SgSymbol* getCopySymbol() const { return copySymbol; }
+    const std::vector<SgStatement*>& getDeclStatements() const { return declStatements; }
+    const std::vector<ParallelRegionLines>& getAllLines() const { return allLines; }
+
+    void addLines(const ParallelRegionLines &newLines)
+    {
+        for (auto &lines : allLines)
+            if (lines == newLines)
+                return;
+
+        allLines.push_back(newLines);
+    }
+
+    void setCopySymbol(SgSymbol *copySymbol) { this->copySymbol = copySymbol; }
+};
+#endif
+
+struct ParallelRegion
+{
+public:
+    ParallelRegion(const uint64_t regionId, const std::string &originalName) : regionId(regionId), originalName(originalName) { }
+
+    ParallelRegion(const ParallelRegion &copy) : allArrays(copy.allArrays), G(copy.G), reducedG(copy.reducedG), dataDirectives(copy.dataDirectives)
+    {
+        regionId = copy.regionId;
+        originalName = copy.originalName;
+        lines = copy.lines;
+        functionsCall = copy.functionsCall;
+        currentVariant = copy.currentVariant;
+    }
+
+    int AddLines(const std::pair<int, int> &linesToAdd, const std::string &file, const std::pair<Statement*, Statement*> *startEnd = NULL)
+    {
+        if (linesToAdd.first > linesToAdd.second)
+            return -1;
+
+        auto it = lines.find(file);
+        if (it == lines.end())
+            it = lines.insert(it, make_pair(file, std::vector<ParallelRegionLines>()));
+
+        if (startEnd)
+            it->second.push_back(ParallelRegionLines(linesToAdd, *startEnd));
+        else
+            it->second.push_back(ParallelRegionLines(linesToAdd));
+
+        return 0;
+    }
+
+    void AddFuncCalls(const std::string &func) { functionsCall.insert(func); }
+
+#if __SPF
+    void AddFuncCallsToAllCalls(FuncInfo *func) { allFunctionsCall.insert(func); }
+#endif
+
+    uint64_t GetId() const { return regionId; }
+    const std::string& GetName() const { return originalName; }
+    const std::map<std::string, std::vector<ParallelRegionLines>>& GetAllLines() const { return lines; }
+    std::map<std::string, std::vector<ParallelRegionLines>>& GetAllLinesToModify() { return lines; }
+    const std::vector<ParallelRegionLines>* GetLines(const std::string &file) const 
+    {
+        auto it = lines.find(file);
+        if (it == lines.end())
+            return NULL;
+        else
+            return &(it->second);
+    }
+    const ParallelRegionLines* GetLinesByLine(const std::string &file, const int line) const
+    {
+        auto fileLines = GetLines(file);
+        if (fileLines)
+            for (auto &lines : *fileLines)
+                if (lines.lines.first <= line && line <= lines.lines.second)
+                    return &lines;
+        return NULL;
+    }
+
+    const DIST::GraphCSR<int, double, attrType>& GetGraph() const { return G; }
+    DIST::GraphCSR<int, double, attrType>& GetGraphToModify() { return G; }
+
+    const DIST::GraphCSR<int, double, attrType>& GetReducedGraph() const { return reducedG; }
+    DIST::GraphCSR<int, double, attrType>& GetReducedGraphToModify() { return reducedG; }
+
+    const DIST::Arrays<int>& GetAllArrays() const { return allArrays; }
+    DIST::Arrays<int>& GetAllArraysToModify() { return allArrays; }
+
+    void SetCurrentVariant(const std::vector<int> &newVariant) { currentVariant = newVariant; }
+    const std::vector<int>& GetCurrentVariant() const { return currentVariant; }
+
+    const DataDirective& GetDataDir() const { return dataDirectives; }
+    DataDirective& GetDataDirToModify() { return dataDirectives; }
+
+    const std::set<std::string>& GetFuncCalls() const { return functionsCall; }
+
+#if __SPF
+    const std::set<FuncInfo*>& GetAllFuncCalls() const { return allFunctionsCall; }
+    const std::map<FuncInfo*, std::map<DIST::Array*, std::vector<ParallelRegionLines>>>& GetUsedLocalArrays() const { return usedLocalArrays; }
+    const std::map<FuncInfo*, std::map<DIST::Array*, std::vector<ParallelRegionLines>>>& GetUsedCommonArrays() const { return usedCommonArrays; }
+
+    void AddUsedLocalArray(FuncInfo *func, DIST::Array *array, const ParallelRegionLines &lines)
+    {
+        auto it = usedLocalArrays.find(func);
+        if (it == usedLocalArrays.end())
+            it = usedLocalArrays.insert(it, std::make_pair(func, std::map<DIST::Array*, std::vector<ParallelRegionLines>>()));
+
+        auto itt = it->second.find(array);
+        if (itt == it->second.end())
+            itt = it->second.insert(itt, std::make_pair(array, std::vector<ParallelRegionLines>()));
+
+        for (auto &curLines : itt->second)
+            if (curLines == lines)
+                return;
+
+        itt->second.push_back(lines);
+    }
+
+    void AddUsedCommonArray(FuncInfo *func, DIST::Array *array, const ParallelRegionLines &lines)
+    {
+        auto it = usedCommonArrays.find(func);
+        if (it == usedCommonArrays.end())
+            it = usedCommonArrays.insert(it, std::make_pair(func, std::map<DIST::Array*, std::vector<ParallelRegionLines>>()));
+
+        auto itt = it->second.find(array);
+        if (itt == it->second.end())
+            itt = it->second.insert(itt, std::make_pair(array, std::vector<ParallelRegionLines>()));
+
+        for (auto &curLines : itt->second)
+            if (curLines == lines)
+                return;
+
+        itt->second.push_back(lines);
+    }
+#endif
+
+    bool HasThisLine(const int line, const std::string &file) const
+    {
+        bool retVal = false;
+        auto it = lines.find(file);
+        if (it != lines.end())
+        {
+            for (int i = 0; i < it->second.size(); ++i)
+            {
+                if (it->second[i].lines.first <= line && it->second[i].lines.second >= line)
+                {
+                    retVal = true;
+                    break;
+                }
+            }
+        }
+
+        return retVal;
+    }
+
+    std::string toString()
+    {
+        std::string retVal = "";
+
+        retVal += "#" + std::to_string(regionId);
+        retVal += "#" + originalName;
+        retVal += "#" + std::to_string(lines.size());
+
+        for (auto it = lines.begin(); it != lines.end(); ++it)
+        {
+            retVal += "|" + it->first + "|";
+            retVal += std::to_string(it->second.size());
+            for (int i = 0; i < it->second.size(); ++i)
+                retVal += "#" + std::to_string(it->second[i].lines.first) + "#" + std::to_string(it->second[i].lines.second);
+        }
+
+        const std::set<DIST::Array*> &arrays = allArrays.GetArrays();
+        retVal += "#" + std::to_string(arrays.size());
+
+        //create map<array_address, DIST::Array_toString()> 
+        for (auto it = arrays.begin(); it != arrays.end(); ++it)
+        {
+            retVal += "#" + std::to_string((long long)(*it));
+            retVal += "#" + (*it)->toString();
+        }
+
+        retVal += "#" + std::to_string(dataDirectives.alignRules.size());
+        for (int i = 0; i < dataDirectives.alignRules.size(); ++i)
+            retVal += dataDirectives.alignRules[i].toString();
+
+        return retVal;
+    }
+
+    void CleanData()
+    {
+        reducedG.ClearGraphCSR();
+        currentVariant.clear();
+
+        dataDirectives.distrRules.clear();
+        dataDirectives.alignRules.clear();
+    }
+
+    void print(FILE *fileOut)
+    {
+        fprintf(fileOut, "  regionId %lld\n", regionId);
+        fprintf(fileOut, "  originalName '%s'\n", originalName.c_str());
+        fprintf(fileOut, "  functions call from %d:\n", (int)functionsCall.size());
+        for (auto &func : functionsCall)
+            fprintf(fileOut, "    '%s'\n", func.c_str());
+        fprintf(fileOut, "  total lines %d:\n", (int)lines.size());
+        for (auto &line : lines)
+        {
+            fprintf(fileOut, "    in file '%s':\n", line.first.c_str());
+            for (auto &elem : line.second)
+            {
+                fprintf(fileOut, "     ");
+                elem.print(fileOut);
+            }
+        }
+    }
+
+    void AddUserDirectives(const std::vector<Statement*> &dirs, const int type)
+    {
+        if (dirs.size() == 0)
+            return;
+
+        if (type == DVM_DISTRIBUTE_DIR || type == DVM_VAR_DECL)
+            userDvmDistrDirs.insert(userDvmDistrDirs.end(), dirs.begin(), dirs.end());
+        else if (type == DVM_ALIGN_DIR)
+            userDvmAlignDirs.insert(userDvmAlignDirs.end(), dirs.begin(), dirs.end());
+        else if (type == DVM_SHADOW_DIR)
+            userDvmShadowDirs.insert(userDvmShadowDirs.end(), dirs.begin(), dirs.end());
+        else if (type == DVM_REALIGN_DIR)
+            userDvmRealignDirs.insert(userDvmRealignDirs.end(), dirs.begin(), dirs.end());
+        else if (type == DVM_REDISTRIBUTE_DIR)
+            userDvmRedistrDirs.insert(userDvmRedistrDirs.end(), dirs.begin(), dirs.end());
+    }
+
+    const std::vector<Statement*>* GetUsersDirecites(const int type) const
+    {
+        if (type == DVM_DISTRIBUTE_DIR || type == DVM_VAR_DECL)
+            return &userDvmDistrDirs;
+        else if (type == DVM_ALIGN_DIR)
+            return &userDvmAlignDirs;
+        else if (type == DVM_SHADOW_DIR)
+            return &userDvmShadowDirs;
+        else if (type == DVM_REALIGN_DIR)
+            return &userDvmRealignDirs;
+        else if (type == DVM_REDISTRIBUTE_DIR)
+            return &userDvmRedistrDirs;
+        else
+            return NULL;
+    }
+
+    bool HasUserDvmDirs() const 
+    {
+        return userDvmDistrDirs.size()   != 0 ||
+               userDvmAlignDirs.size()   != 0 ||
+               userDvmShadowDirs.size()  != 0 ||
+               userDvmRealignDirs.size() != 0 ||
+               userDvmRedistrDirs.size() != 0;
+    }
+#if __SPF
+    void ClearUserDirs()
+    {
+        ClearVector(userDvmDistrDirs);
+        ClearVector(userDvmAlignDirs);
+        ClearVector(userDvmShadowDirs);
+        ClearVector(userDvmRealignDirs);
+        ClearVector(userDvmRedistrDirs);
+    }
+#endif
+private:
+    uint64_t regionId;
+    //name in program
+    std::string originalName;
+    // file -> lines info
+    std::map<std::string, std::vector<ParallelRegionLines>> lines;
+    std::set<std::string> functionsCall;
+
+#if __SPF
+    // for RESOLVE_PAR_REGIONS
+    std::set<FuncInfo*> allFunctionsCall;
+    std::map<FuncInfo*, std::map<DIST::Array*, std::vector<ParallelRegionLines>>> usedLocalArrays;  // func -> array -> lines
+    std::map<FuncInfo*, std::map<DIST::Array*, std::vector<ParallelRegionLines>>> usedCommonArrays; // func -> array -> lines
+    //
+#endif
+
+    // for LOOP_ANALYZER_DATA_DIST
+    DIST::GraphCSR<int, double, attrType> G;
+    DIST::Arrays<int> allArrays;
+
+    DIST::GraphCSR<int, double, attrType> reducedG;
+    //
+
+    //for directive creating
+    DataDirective dataDirectives;
+    std::vector<int> currentVariant;
+    //
+
+    std::vector<Statement*> userDvmDistrDirs;
+    std::vector<Statement*> userDvmAlignDirs;
+    std::vector<Statement*> userDvmShadowDirs;
+    std::vector<Statement*> userDvmRealignDirs;
+    std::vector<Statement*> userDvmRedistrDirs;
+
+#if __SPF
+    void ClearVector(std::vector<Statement*> &toRem)
+    {
+        for (auto& elem : toRem)
+        {
+            if (SgFile::switchToFile(elem->fileName()))
+                ;// printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+            else
+                elem->deleteStmt();
+        }
+        toRem.clear();
+    }
+#endif
+};
+
+ParallelRegion* getRegionById(const std::vector<ParallelRegion*>& regions, const uint64_t regionId);
+ParallelRegion* getRegionByName(const std::vector<ParallelRegion*>& regions, const std::string& regionName);
+ParallelRegion* getRegionByLine(const std::vector<ParallelRegion*>& regions, const std::string& file, const int line);
+std::set<ParallelRegion*> getAllRegionsByLine(const std::vector<ParallelRegion*>& regions, const std::string& file, const int line);

src/ParallelizationRegions/ParRegions_func.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "ParRegions.h"
+#include "../GraphCall/graph_calls.h"
+#include "../GraphLoop/graph_loops.h"
+
+void fillRegionLines(SgFile *file, std::vector<ParallelRegion*> &regions, std::vector<Messages>& messagesForFile, std::vector<LoopGraph*> *loops = NULL, std::vector<FuncInfo*> *funcs = NULL);
+void fillRegionLinesStep2(std::vector<ParallelRegion*> &regions, const std::map<std::string, std::vector<FuncInfo*>> &allFuncInfo, std::map<std::string, std::vector<LoopGraph*>> *loopGraph = NULL);
+int printParalleRegions(const char *fileName, std::vector<ParallelRegion*> &regions);
+bool buildGraphFromUserDirectives(const std::vector<Statement*> &userDvmAlignDirs, DIST::GraphCSR<int, double, attrType> &G, DIST::Arrays<int> &allArrays, const std::map<DIST::Array*, std::set<DIST::Array*>> &arrayLinksByFuncCalls, const std::set<DIST::Array*>& alignedArrays, std::set<DIST::Array*>& addedArrays, const std::map<std::string, std::vector<FuncInfo*>>& funcsByFile);
+void clearRegionStaticData();
+void calculateLinesOfCode(std::vector<ParallelRegion*> &allRegions);

src/ParallelizationRegions/expand_extract_reg.cpp

@@ -0,0 +1,487 @@
+#include "../Utils/leak_detector.h"
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "ParRegions_func.h"
+#include "expand_extract_reg.h"
+
+using std::map;
+using std::pair;
+using std::vector;
+using std::string;
+using std::to_string;
+using std::make_pair;
+
+static inline void insertParRegDir(SgStatement *start, const string &regName)
+{
+    // !SPF PARALLEL_REG regName
+    SgStatement *regSt = new SgStatement(SPF_PARALLEL_REG_DIR);
+    checkNull(start->getFile(), convertFileName(__FILE__).c_str(), __LINE__);
+
+    SgSymbol *regSymb =  findSymbolOrCreate(start->getFile(), regName.c_str());
+    regSt->setSymbol(*regSymb);
+    start->insertStmtBefore(*regSt, *start->controlParent());
+}
+
+static inline void insertEndParReg(SgStatement *end)
+{
+    // !SPF END PARALLEL_REG
+    SgStatement *regSt = new SgStatement(SPF_END_PARALLEL_REG_DIR);
+    SgStatement *next = end->lexNext();
+    checkNull(next, convertFileName(__FILE__).c_str(), __LINE__);
+    next->insertStmtBefore(*regSt, *next->controlParent());
+}
+
+static inline void insertParRegDirs(SgStatement *start, SgStatement *end, const string &regName)
+{
+    insertParRegDir(start, regName);
+    insertEndParReg(end);
+}
+
+static bool checkIfLineIsImplicit(const ParallelRegion *reg, const string &fileName, const int line)
+{
+    if (reg && reg->GetName() != "DEFAULT")
+    {
+        auto regLines = reg->GetLinesByLine(fileName, line);
+        if (regLines && regLines->isImplicit())
+            return true; // error
+    }
+    return false;
+}
+
+static bool hasOwnControlParent(SgStatement *start, SgStatement *end)
+{
+    if (start == end)
+        return true;
+
+    if (!end->controlParent())
+        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+    return (end->variant() != CONTROL_END && start->controlParent() == end->controlParent() ||
+            end->variant() == CONTROL_END && start->controlParent() == end->controlParent()->controlParent());
+}
+
+static bool isNotComposit(SgStatement *st)
+{
+    if (!st)
+        return true;
+    return (st->lastNodeOfStmt() == st);
+}
+
+bool expandExtractReg(const string &fileName,
+                      int startLine,
+                      int endLine,
+                      const vector<ParallelRegion*> &regions,
+                      vector<Messages> &messagesForFile,
+                      const bool toDelete)
+{
+    bool error = false;
+
+    if (SgFile::switchToFile(fileName) != -1)
+    {
+        SgStatement *begin = SgStatement::getStatementByFileAndLine(fileName, startLine);
+        SgStatement *end = SgStatement::getStatementByFileAndLine(fileName, endLine);
+
+        int lastLine = -1;
+        for (auto st = current_file->firstStatement(); st; st = st->lexNext())
+            lastLine = st->lineNumber();
+
+        for (; !begin && startLine <= lastLine;)
+            begin = SgStatement::getStatementByFileAndLine(fileName, ++startLine);
+
+        for (; !end && endLine > 0;)
+            end = SgStatement::getStatementByFileAndLine(fileName, --endLine);
+
+        // check user lines
+        // 2. <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> N1 <20> N2 <20> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+        int errorLine;
+        bool localError = false;
+
+        if (!(isSgExecutableStatement(end) || end->variant() == ENTRY_STAT))
+        {
+            localError = true;
+            errorLine = endLine;
+        }
+
+        if (!(isSgExecutableStatement(begin) || begin->variant() == ENTRY_STAT))
+        {
+            localError = true;
+            errorLine = startLine;
+        }
+        
+        if (localError)
+        {
+            __spf_print(1, "bad directive position on line %d: it can be placed only after all DATA statements\n", errorLine);
+            
+            std::wstring bufwE, bufwR;
+            __spf_printToLongBuf(bufwE, L"bad directive position: it can be placed only after all DATA statements");
+            __spf_printToLongBuf(bufwR, R12);
+            messagesForFile.push_back(Messages(ERROR, errorLine, bufwR, bufwE, 1001));
+
+            error = true;
+        }
+
+        // 1. N1 <= N2
+        if (startLine > endLine)
+        {
+            __spf_print(1, "bad lines position: end line %d must be greater or equel begin line %d\n", endLine, startLine);
+
+            std::wstring bufwE, bufwR;
+            __spf_printToLongBuf(bufwE, L"bad lines position: end line %d must be greater or equel begin line %d", endLine, startLine);
+            __spf_printToLongBuf(bufwR, R13);
+            messagesForFile.push_back(Messages(ERROR, endLine, bufwR, bufwE, 1001));
+
+            error = true;
+        }
+
+        // 2. <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> N1 <20> N2 <20><> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><> <20> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+        auto beginReg = getRegionByLine(regions, fileName, startLine);
+        auto endReg = getRegionByLine(regions, fileName, endLine);
+
+        if (!toDelete && beginReg && endReg && beginReg != endReg)
+        {
+            __spf_print(1, "bad lines position: begin and end lines can not be placed at differect regions\n");
+
+            std::wstring bufwE, bufwR;
+            __spf_printToLongBuf(bufwE, L"bad lines position: begin and end lines can not be placed at differect regions");
+            __spf_printToLongBuf(bufwR, R14);
+            messagesForFile.push_back(Messages(ERROR, endLine, bufwR, bufwE, 1001));
+
+            error = true;
+        }
+
+        localError = false;
+
+        if (checkIfLineIsImplicit(endReg, fileName, endLine))
+        {
+            localError = true;
+            errorLine = endLine;
+        }
+
+        if (checkIfLineIsImplicit(beginReg, fileName, startLine))
+        {
+            localError = true;
+            errorLine = startLine;
+        }
+
+        if (localError)
+        {
+            __spf_print(1, "bad lines position on line %d: begin and end lines can not be placed at region implicit lines\n", errorLine);
+
+            std::wstring bufwE, bufwR;
+            __spf_printToLongBuf(bufwE, L"bad lines position: begin and end lines can not be placed at region implicit lines");
+            __spf_printToLongBuf(bufwR, R15);
+            messagesForFile.push_back(Messages(ERROR, errorLine, bufwR, bufwE, 1001));
+
+            error = true;
+        }
+
+        // X. <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+        SgStatement *beginFunc = begin, *endFunc = end;
+
+        while (!isSgProgHedrStmt(beginFunc))
+            beginFunc = beginFunc->controlParent();
+
+        while (!isSgProgHedrStmt(endFunc))
+            endFunc = endFunc->controlParent();
+
+        if (beginFunc != endFunc)
+        {
+            __spf_print(1, "bad lines position: begin and end lines can not be placed at different functions\n");
+
+            std::wstring bufwE, bufwR;
+            __spf_printToLongBuf(bufwE, L"bad lines position: begin and end lines can not be placed at different functions");
+            __spf_printToLongBuf(bufwR, R16);
+            messagesForFile.push_back(Messages(ERROR, errorLine, bufwR, bufwE, 1001));
+
+            error = true;
+        }
+
+        // 3. <20><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> [N1,N2] <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><>
+        const ParallelRegionLines *beginLines = NULL, *endLines = NULL;
+        map<const ParallelRegion*, vector<const ParallelRegionLines*>> internalLines;
+
+        for (auto &reg : regions)
+        {
+            auto regLines = reg->GetLines(fileName);
+
+            for (auto &lines : *regLines)
+            {
+                if (!lines.isImplicit())
+                {
+                    /*
+                    if (startLine >= lines.lines.first && startLine <= lines.lines.second)
+                        beginLines = &lines;
+
+                    if (endLine >= lines.lines.first && endLine <= lines.lines.second)
+                        endLines = &lines;
+                    */
+                    if (startLine < lines.lines.first && endLine > lines.lines.second)
+                    {
+                        auto it = internalLines.find(reg);
+                        if (it == internalLines.end())
+                            it = internalLines.insert(it, make_pair(reg, vector<const ParallelRegionLines*>()));
+                        it->second.push_back(&lines);
+                    }
+                }
+            }
+        }
+
+        if (!toDelete && internalLines.size() > 1)
+        {
+            __spf_print(1, "bad lines position on line %d: begin and end lines can not include fragments of different regions at extending operation\n", startLine);
+
+            std::wstring bufwE, bufwR;
+            __spf_printToLongBuf(bufwE, L"bad lines position: begin and end lines can not include fragments of different regions at extending operation");
+            __spf_printToLongBuf(bufwR, R17);
+            messagesForFile.push_back(Messages(ERROR, startLine, bufwR, bufwE, 1001));
+
+            error = true;
+        }
+
+        // use cases
+        if (beginReg && beginReg->GetName() != "DEFAULT")
+            beginLines = beginReg->GetLinesByLine(fileName, startLine);
+
+        if (endReg && endReg->GetName() != "DEFAULT")
+            endLines = endReg->GetLinesByLine(fileName, endLine);
+
+        // delete all internal fragments
+        for (auto &regLines : internalLines)
+        {
+            for (auto &lines : regLines.second)
+            {
+                lines->stats.first->GetOriginal()->lexPrev()->deleteStmt();
+                lines->stats.second->GetOriginal()->lexNext()->deleteStmt();
+            }
+        }
+
+        if (error)
+            return true;
+
+        localError = false;
+
+        if (beginLines && !endLines)
+        {
+            if (!toDelete)
+            {
+                if (hasOwnControlParent(beginLines->stats.first->GetOriginal(), end))
+                {
+                    insertEndParReg(end);
+                    beginLines->stats.second->GetOriginal()->lexNext()->deleteStmt();
+                }
+                else
+                {
+                    errorLine = endLine;
+                    localError = true;
+                }
+            }
+            else
+            {
+                if (hasOwnControlParent(beginLines->stats.first->GetOriginal(), begin) && begin->variant() != CONTROL_END)
+                {
+                    if (startLine == beginLines->lines.first)
+                        beginLines->stats.first->GetOriginal()->lexPrev()->deleteStmt();
+                    else
+                        insertEndParReg(begin->lexPrev());
+                    beginLines->stats.second->GetOriginal()->lexNext()->deleteStmt();
+                }
+                else
+                {
+                    errorLine = startLine;
+                    localError = true;
+                }
+            }
+        }
+        else if (endLines && !beginLines)
+        {
+            if (!toDelete)
+            {
+                if (hasOwnControlParent(begin, endLines->stats.first->GetOriginal()))
+                {
+                    insertParRegDir(begin, endReg->GetName());
+                    endLines->stats.first->GetOriginal()->lexPrev()->deleteStmt();
+                }
+                else
+                {
+                    errorLine = startLine;
+                    localError = true;
+                }
+            }
+            else
+            {
+                if (hasOwnControlParent(endLines->stats.first->GetOriginal(), end) && isNotComposit(end))
+                {
+                    if (endLine == endLines->lines.second)
+                        endLines->stats.second->GetOriginal()->lexNext()->deleteStmt();
+                    else
+                        insertParRegDir(end->lexNext(), endReg->GetName());
+                    endLines->stats.first->GetOriginal()->lexPrev()->deleteStmt();
+                }
+                else
+                {
+                    errorLine = endLine;
+                    localError = true;
+                }
+            }
+        }
+        else if (!beginLines && !endLines)
+        {
+            if (!toDelete)
+            {
+                string regName;
+                // TODO: create correct new region name
+                if (internalLines.size())
+                    regName = internalLines.begin()->first->GetName();
+                else
+                    regName = "reg" + to_string(regions.size());
+
+                if (hasOwnControlParent(begin, end))
+                    insertParRegDirs(begin, end, regName);
+                else
+                {
+                    __spf_print(1, "bad lines %d-%d position: expected lines with the same scope for creating region fragment\n", startLine, endLine);
+
+                    std::wstring bufwE, bufwR;
+                    __spf_printToLongBuf(bufwE, L"bad lines position: expected lines with the same scope for creating region fragment");
+                    __spf_printToLongBuf(bufwR, R18);
+                    messagesForFile.push_back(Messages(ERROR, endLine, bufwR, bufwE, 1001));
+
+                    error = true;
+                }
+            }
+        }
+        else if (beginLines && endLines)
+        {
+            if (!toDelete)
+            {
+                if (beginLines != endLines && hasOwnControlParent(beginLines->stats.first->GetOriginal(), endLines->stats.first->GetOriginal()))
+                {
+                    beginLines->stats.second->GetOriginal()->lexNext()->deleteStmt();
+                    endLines->stats.first->GetOriginal()->lexPrev()->deleteStmt();
+                }
+                else
+                {
+                    __spf_print(1, "bad lines %d-%d position: can not extend region fragments with different scope\n", startLine, endLine);
+
+                    std::wstring bufwE, bufwR;
+                    __spf_printToLongBuf(bufwE, L"bad lines %d-%d position: can not extend region fragments with different scope", startLine, endLine);
+                    __spf_printToLongBuf(bufwR, R19, startLine, endLine);
+                    messagesForFile.push_back(Messages(ERROR, endLine, bufwR, bufwE, 1001));
+
+                    error = true;
+                }
+            }
+            else
+            {
+                int errorLine1, errorLine2;
+                localError = false;
+
+                /*
+                if (beginLines == endLines && !hasOwnControlParent(begin, end))
+                {
+                    localError = true;
+                    errorLine = startLine;
+                    errorLine1 = startLine;
+                    errorLine2 = endLine;
+                }
+                */
+
+                if (hasOwnControlParent(beginLines->stats.first->GetOriginal(), begin) && begin->variant() != CONTROL_END)
+                {
+                    if (startLine == beginLines->lines.first)
+                        beginLines->stats.first->GetOriginal()->lexPrev()->deleteStmt();
+                    else
+                        insertEndParReg(begin->lexPrev());
+                }
+                else
+                {
+                    localError = true;
+                    errorLine = startLine;
+                    errorLine1 = beginLines->lines.first;
+                    errorLine2 = startLine;
+                }
+
+                if (hasOwnControlParent(endLines->stats.first->GetOriginal(), end) && isNotComposit(end))
+                {
+                    if (endLine == endLines->lines.second)
+                        endLines->stats.second->GetOriginal()->lexNext()->deleteStmt();
+                    else
+                        insertParRegDir(end->lexNext(), endReg->GetName());
+                }
+                else
+                {
+                    localError = true;
+                    errorLine = endLine;
+                    errorLine1 = endLine;
+                    errorLine2 = endLines->lines.second;
+                }
+
+                if (beginLines != endLines)
+                {
+                    beginLines->stats.second->GetOriginal()->lexNext()->deleteStmt();
+                    endLines->stats.first->GetOriginal()->lexPrev()->deleteStmt();
+                }
+                /*
+                if (localError)
+                {
+                    __spf_print(1, "bad lines %d-%d position: expected line with the same scope for extracting region fragment\n", errorLine1, errorLine2);
+
+                    std::wstring bufw;
+                    __spf_printToLongBuf(bufw, L"bad lines position: expected line with the same scope for extracting region fragment");
+                    messagesForFile.push_back(Messages(ERROR, errorLine, bufw, 1001));
+
+                    error = true;
+                }
+                */
+            }
+        }
+
+        if (localError)
+        {
+            __spf_print(1, "bad lines position: expected lines with the same control parent on line %d\n", errorLine);
+
+            std::wstring bufwE, bufwR;
+            __spf_printToLongBuf(bufwE, L"bad lines position: expected lines with the same control parent");
+            __spf_printToLongBuf(bufwR, R20);
+            messagesForFile.push_back(Messages(ERROR, errorLine, bufwR, bufwE, 1001));
+
+            error = true;
+        }
+
+        // TODO: check: delete all possible empty fragments
+        /*
+        SgStatement *parStart = NULL;
+        SgStatement *parEnd = NULL;
+
+        while (begin->variant() != PROG_HEDR && begin->variant() != PROC_HEDR && begin->variant() != FUNC_HEDR)
+            begin = begin->controlParent();
+        end = begin->lastNodeOfStmt();
+
+        for (auto st = begin; st != end; st = st->lexNext())
+        {
+            if (st->variant() == SPF_PARALLEL_REG_DIR)
+                parStart = st;
+
+            if (st->variant() == SPF_END_PARALLEL_REG_DIR)
+            {
+                parEnd = st;
+
+                if (parStart && parStart->lexNext() == parEnd)
+                {
+                    parStart->deleteStmt();
+                    parEnd->deleteStmt();
+                    parStart = NULL;
+                    parEnd = NULL;
+                }
+            }
+        }
+        */
+    }
+    else
+        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+    return error;
+}

src/ParallelizationRegions/expand_extract_reg.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include "ParRegions.h"
+#include "../Utils/SgUtils.h"
+#include "../Utils/errors.h"
+
+bool expandExtractReg(const std::string &fileName, const int startLine, const int endLine, const std::vector<ParallelRegion*> &regions, std::vector<Messages> &messagesForFile, const bool toDelete = false);

src/ParallelizationRegions/resolve_par_reg_conflicts.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include "ParRegions.h"
+#include "../Utils/SgUtils.h"
+#include "../Utils/errors.h"
+#include "../GraphCall/graph_calls.h"
+
+void fillRegionIntervals(std::vector<ParallelRegion*> &regions);
+void fillRegionArrays(std::vector<ParallelRegion*> &regions, const std::map<std::string, std::vector<FuncInfo*>> &allFuncInfo, const std::map<std::string, CommonBlock*> &commonBlocks);
+void fillRegionFunctions(std::vector<ParallelRegion*> &regions, const std::map<std::string, std::vector<FuncInfo*>> &allFuncInfo);
+bool checkRegions(const std::vector<ParallelRegion*> &regions, const std::map<std::string, std::vector<FuncInfo*>> &allFuncInfo, std::map<std::string, std::vector<Messages>> &SPF_messages);
+int printCheckRegions(const char *fileName, const std::vector<ParallelRegion*> &regions, const std::map<std::string, std::vector<FuncInfo*>> &allFuncInfo);
+
+bool checkRegionsResolving(const std::vector<ParallelRegion*> &regions, const std::map<std::string, std::vector<FuncInfo*>> &allFuncInfo, const std::map<std::string, CommonBlock*> &commonBlocks, std::map<std::string, std::vector<Messages>> &SPF_messages, bool sharedMemoryParallelization);
+int resolveParRegions(std::vector<ParallelRegion*>& regions, const std::map<std::string, std::vector<FuncInfo*>>& allFuncInfo, std::map<std::string, std::vector<Messages>>& SPF_messages, bool sharedMemoryParallelization, std::map<std::string, std::map<int, std::set<std::string>>>& copyDecls);
+void insertRealignsBeforeFragments(ParallelRegion* reg, SgFile* file, const std::set<DIST::Array*>& distrArrays, const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls);
+
+std::pair<SgSymbol*, SgSymbol*> copyArray(const std::pair<std::string, int>& place,
+                                          const DIST::Array* array,
+                                          const std::vector<ParallelRegionLines>& lines,
+                                          const std::string& suffix,
+                                          std::string& filename,
+                                          std::map<std::string, std::map<int, std::set<std::string>>>& newDeclsToInclude,
+                                          std::map<std::string, std::map<int, std::set<std::string>>>& copied);