SAPFOR/Sapfor/_src/DirectiveProcessing/shadow.cpp

#include "../Utils/leak_detector.h"

#include <set>
#include <map>
#include <string>
#include <vector>

#include "../Utils/errors.h"
#include "../Utils/utils.h"
#include "../GraphLoop/graph_loops_func.h"
#include "../GraphCall/graph_calls.h"
#include "../GraphCall/graph_calls_func.h"
#include "directive_parser.h"
#include "../Distribution/DvmhDirective_func.h"
#include "../Utils/SgUtils.h"
#include "../ExpressionTransform/expr_transform.h"

#include "../CFGraph/CFGraph.h"

#include "shadow.h"
#include "dvm.h"

using std::set;
using std::map;
using std::string;
using std::vector;
using std::pair;
using std::make_pair;

extern int debSh;

static void findShadowAndRemote(SgExpression *spec, SgExpression *&shadow, SgExpression *&remote, SgExpression *&beforeSh)
{
    remote = shadow = NULL;
    beforeSh = spec;

    for (auto iter = spec, iterB = spec; iter; iter = iter->rhs())
    {
        if (iter->lhs()->variant() == SHADOW_RENEW_OP)
        {
            beforeSh = iterB;
            shadow = iter->lhs();
        }
        else if (iter->lhs()->variant() == REMOTE_ACCESS_OP)
            remote = iter->lhs();

        if (iterB != iter)
            iterB = iterB->rhs();
    }
}

static set<string> getAllRemoteWithDDOT(SgExpression *remote)
{
    set<string> allRemoteWithDDOT;
    for (auto iter = remote->lhs(); iter; iter = iter->rhs())
    {
        SgExpression *elem = iter->lhs();
        if (elem->variant() == ARRAY_REF)
        {
            bool allDDOT = true;
            for (auto iterL = elem->lhs(); iterL; iterL = iterL->rhs())
                if (iterL->lhs()->variant() != DDOT)
                    allDDOT = false;

            if (allDDOT)
                allRemoteWithDDOT.insert(elem->symbol()->identifier());
        }
    }

    return allRemoteWithDDOT;
}

static DIST::Array* getArrayFromAttribute(SgExpression *elem)
{
    DIST::Array *currArray = NULL;
    for (int i = 0; i < elem->numberOfAttributes() && currArray == NULL; ++i)
        if (elem->attributeType(i) == ARRAY_REF)
            currArray = (DIST::Array *)(elem->getAttribute(i)->getAttributeData());

    if (currArray == NULL)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    return currArray;
}


static vector<pair<int, int>> fillShadowSpec(SgExpression *elem)
{
    vector<pair<int, int>> toDel;
    for (SgExpression *list = elem->lhs(); list; list = list->rhs())
        toDel.push_back(make_pair(list->lhs()->lhs()->valueInteger(), list->lhs()->rhs()->valueInteger()));

    return toDel;
}

static void devourShadow(SgExpression *spec, SgStatement *stat)
{
    if (spec)
    {
        SgExpression *shadow, *remote, *beforeSh;
        findShadowAndRemote(spec, shadow, remote, beforeSh);
        
        if (shadow && remote)
        {
            const set<string> allRemoteWithDDOT = getAllRemoteWithDDOT(remote);

            auto currShadowP = shadow;
            int numActiveSh = 0;

            for (auto iter = shadow->lhs(); iter; iter = iter->rhs())
            {
                SgExpression *elem = iter->lhs();
                //if shadow has CORNER
                if (elem->variant() == ARRAY_OP)
                    elem = elem->lhs();

                if (elem->variant() == ARRAY_REF)
                {
                    if (allRemoteWithDDOT.find(elem->symbol()->identifier()) != allRemoteWithDDOT.end())
                    {
                        DIST::Array *currArray = getArrayFromAttribute(elem);
                        vector<pair<int, int>> toDel = fillShadowSpec(elem); 

                        currArray->RemoveShadowSpec(toDel);
                        if (currShadowP == shadow)
                            shadow->setLhs(iter->rhs());
                        else
                            currShadowP->setRhs(iter->rhs());
                    }
                    else
                    {
                        ++numActiveSh;
                        if (currShadowP == shadow)
                            currShadowP = shadow->lhs();
                        else
                            currShadowP = currShadowP->rhs();
                    }
                }
            }

            //remove shadow dir
            if (numActiveSh == 0)
            {
                if (spec->lhs()->variant() == SHADOW_RENEW_OP)
                    stat->setExpression(1, *(spec->rhs()));
                else
                    beforeSh->setRhs(beforeSh->rhs()->rhs());
            }
        }
    }
}

extern vector<bool> isSimpleRef(SgStatement* stS, SgStatement* stE, SgExpression* subs, const set<int> noSimpleVars, const map<string, FuncInfo*>& funcMap, const set<string>& usedVars);
static void convertShadowToDDOTRemote(SgExpression *spec, const LoopGraph* loop, const map<string, FuncInfo*>& funcMap)
{
    //TODO: need to optimize
    //set<string> loopVars(loop->directive->parallel.begin(), loop->directive->parallel.end());
    //vector<bool> isSimple = isSimpleRef(loop->loop->GetOriginal(), loop->loop->GetOriginal()->lastNodeOfStmt(), spec, { ARRAY_OP, ARRAY_REF, VAR_REF }, funcMap, loopVars);

    while (spec)
    {
        spec->setLhs(new SgExpression(DDOT));
        spec = spec->rhs();
    }
}

extern int maxShadowWidth;
extern SgExpression* remoteAggregation(SgExpression* remote, const vector<SgExpression*>* newRemotes);
static void replaceShadowByRemote(SgExpression *specInDir, SgStatement *stat,
                                  const map<DIST::Array*, set<DIST::Array*>> &arrayLinksByFuncCalls, 
                                  const LoopGraph* loop, const map<string, FuncInfo*>& funcMap)
{
    if (specInDir)
    {
        SgExpression *shadow, *remote, *beforeSh;
        findShadowAndRemote(specInDir, shadow, remote, beforeSh);

        if (shadow)
        {            
            set<string> remotesNames;

            SgExpression *newRemote = NULL;
            SgExpression *pRem = NULL;

            map<pair<string, string>, Expression*> remotes;

            if (!remote)
                pRem = newRemote = new SgExpression(REMOTE_ACCESS_OP);
            else
            {
                fillRemoteFromComment(new Statement(stat), remotes, true, DVM_PARALLEL_ON_DIR);
                for (auto &elem : remotes)
                    remotesNames.insert(elem.first.first);
                pRem = remote;
            }
            
            bool remoteWasAdded = false;
            auto currShadowP = shadow;
            int numActiveSh = 0;

            for (auto iter = shadow->lhs(); iter; iter = iter->rhs())
            {
                SgExpression *elem = iter->lhs();
                //if shadow has CORNER
                if (elem->variant() == ARRAY_OP)
                    elem = elem->lhs();

                if (elem->variant() == ARRAY_REF)
                {
                    DIST::Array *currArray = getArrayFromAttribute(elem);
                    vector<pair<int, int>> spec = fillShadowSpec(elem);
                    

                    set<DIST::Array*> realRefs;
                    getRealArrayRefs(currArray, currArray, realRefs, arrayLinksByFuncCalls);

                    bool replaceByRemote = false;
                    for (auto &realArray : realRefs)
                    {
                        auto arraySizes = realArray->GetSizes();
                        //check sizes
                        for (auto &dim : arraySizes)
                            if (dim.first > dim.second || (dim.first == -1 && dim.second == -1))
                                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

                        if (spec.size() != arraySizes.size())
                            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
                                                
                        for (int z = 0; z < spec.size(); ++z)
                        {
                            float maxSpec = std::max(spec[z].first, spec[z].second);
                            float dimSize = arraySizes[z].second - arraySizes[z].first + 1;

                            // 50 % by default
                            if (dimSize * (maxShadowWidth / 100.) < maxSpec)
                            {
                                replaceByRemote = true;
                                break;
                            }
                        }

                        if (replaceByRemote)
                            break;
                    }

                    if (replaceByRemote)
                    {
                        __spf_print(1, "RemoteAccess[devour]: replace shadow by remote for array '%s'\n", currArray->GetShortName().c_str());
                        currArray->RemoveShadowSpec(spec);
                        for (auto& realArray : realRefs)
                            realArray->RemoveShadowSpec(spec);

                        auto keyName = OriginalSymbol(elem->symbol())->identifier();
                        auto it = remotesNames.find(keyName);
                        if (it == remotesNames.end())
                        {
                            remotesNames.insert(it, OriginalSymbol(elem->symbol())->identifier());

                            SgExpression* toAdd = new SgExpression(EXPR_LIST);
                            toAdd->setLhs(elem);
                            toAdd->setRhs(pRem->lhs());
                            pRem->setLhs(toAdd);

                            convertShadowToDDOTRemote(elem->lhs(), loop, funcMap);
                        }
                        else 
                        {                            
                            for (auto& elem : remotes)
                                if (elem.first.first == keyName)
                                    convertShadowToDDOTRemote(elem.second->GetOriginal()->lhs(), loop, funcMap);
                        }                     

                        remoteWasAdded = true;
                        if (currShadowP == shadow)
                            shadow->setLhs(iter->rhs());
                        else
                            currShadowP->setRhs(iter->rhs());
                    }
                    else
                    {
                        ++numActiveSh;
                        if (currShadowP == shadow)
                            currShadowP = shadow->lhs();
                        else
                            currShadowP = currShadowP->rhs();
                    }
                }
            }

            //remove shadow dir
            if (numActiveSh == 0)
            {
                if (specInDir->lhs()->variant() == SHADOW_RENEW_OP)
                    stat->setExpression(1, *(specInDir->rhs()));
                else
                    beforeSh->setRhs(beforeSh->rhs()->rhs());
            }

            if (remoteWasAdded)
            {
                if (newRemote)
                    stat->setExpression(1, *new SgExpression(EXPR_LIST, newRemote, stat->expr(1), NULL));
                else                    
                    remoteAggregation(remote, NULL);
            }            
        }
    }
}

void devourShadowByRemote(void *file_, const map<string, FuncInfo*>& funcMap, const vector<LoopGraph*>& loops, 
                          const map<DIST::Array*, set<DIST::Array*>> &arrayLinksByFuncCalls)
{
    SgFile* file = static_cast<SgFile*>(file_);
    map<int, LoopGraph*> loopByLine;
    createMapLoopGraph(loops, loopByLine);

    for (SgStatement *stat = file->firstStatement(); stat; stat = stat->lexNext())
    {
        if (stat->variant() == DVM_PARALLEL_ON_DIR && stat->lineNumber() == 0) // except user dirs
        {
            devourShadow(stat->expr(1), stat);
            auto next = stat->lexNext();
            if (next->variant() != FOR_NODE)
                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

            if (loopByLine.find(next->lineNumber()) == loopByLine.end())
                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

            replaceShadowByRemote(stat->expr(1), stat, arrayLinksByFuncCalls, loopByLine[next->lineNumber()], funcMap);
        }
    }
}

static void transformShadowIfFull(SgExpression* shadowList, const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls)
{
    if (shadowList)
    {
        for (auto iter = shadowList->lhs(); iter; iter = iter->rhs())
        {
            SgExpression* elem = iter->lhs();

            //if shadow has CORNER
            if (elem->variant() == ARRAY_OP)
                elem = elem->lhs();

            if (elem->variant() == ARRAY_REF)
            {
                DIST::Array* currArray = NULL;
                for (int i = 0; i < elem->numberOfAttributes() && currArray == NULL; ++i)
                    if (elem->attributeType(i) == ARRAY_REF)
                        currArray = (DIST::Array*)(elem->getAttribute(i)->getAttributeData());

                if (currArray == NULL)
                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

                set<DIST::Array*> arrays;
                getRealArrayRefs(currArray, currArray, arrays, arrayLinksByFuncCalls);

                if (arrays.size() == 0)
                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

                auto shadowSpec = (*arrays.begin())->GetShadowSpec();
                if (shadowSpec.size() != currArray->GetDimSize())
                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

                SgArrayRefExp* ref = (SgArrayRefExp*)elem;
                if (ref->numberOfSubscripts() != currArray->GetDimSize())
                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

                bool eq = true;
                for (int i = 0; i < currArray->GetDimSize(); ++i)
                {
                    SgExpression* subs = ref->subscript(i);
                    if (subs->variant() == DDOT)
                    {
                        if (subs->lhs()->isInteger() && subs->rhs()->isInteger())
                        {
                            if (subs->lhs()->valueInteger() != shadowSpec[i].first ||
                                subs->rhs()->valueInteger() != shadowSpec[i].second)
                            {
                                eq = false;
                                break;
                            }
                        }
                        else
                        {
                            eq = false;
                            break;
                        }
                    }
                    else
                    {
                        eq = false;
                        break;
                    }
                }

                //remove
                if (eq)
                    elem->setLhs(NULL);
            }
        }
    }
}

void transformShadowIfFull(void *file_, const map<DIST::Array*, set<DIST::Array*>> &arrayLinksByFuncCalls)
{
    SgFile* file = static_cast<SgFile*>(file_);
    for (SgStatement *first = file->firstStatement(); first; first = first->lexNext())
    {
        if (first->variant() == DVM_PARALLEL_ON_DIR && first->lineNumber() == 0) // except user dirs
        {
            SgExpression *spec = first->expr(1);
            if (spec)
            {
                SgExpression *shadowList = NULL;
                for (auto iter = spec; iter; iter = iter->rhs())
                {
                    if (iter->lhs()->variant() == SHADOW_RENEW_OP)
                    {
                        shadowList = iter->lhs();
                        break;
                    }
                }
                transformShadowIfFull(shadowList, arrayLinksByFuncCalls);
            }
        }
    }
}

static void addRealArraysRef(set<DIST::Array*>& writesTo, DIST::Array* array,
                             const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls)
{
    set<DIST::Array*> realRef;
    getRealArrayRefs(array, array, realRef, arrayLinksByFuncCalls);

    for (auto& realR : realRef)
        writesTo.insert(realR);
}

static void findFuncCalls(SgExpression* ex, vector<NextNode>& next, 
                          const map<void*, ShadowNode*>& allShadowNodes,
                          const set<DIST::Array*>& writesTo)
{
    if (ex)
    {
        if (ex->variant() == FUNC_CALL)
        {
            if (isIntrinsicFunctionName(ex->symbol()->identifier()) == false)
                if (allShadowNodes.find(ex) != allShadowNodes.end())
                    next.push_back(NextNode(allShadowNodes.find(ex)->second, writesTo));
        }

        findFuncCalls(ex->lhs(), next, allShadowNodes, writesTo);
        findFuncCalls(ex->rhs(), next, allShadowNodes, writesTo);
    }
}

static void viewGraph(CBasicBlock *first)
{
    auto nexts = first->getSucc();
    auto items = first->getStart();
    auto itemsEnd = first->getEnd();
    while (items != itemsEnd)
    {
        auto stmt = items->getStatement();
        if (stmt)
            printf("%d tag %s\n", stmt->lineNumber(), tag[stmt->variant()]);

        stmt = items->getOriginalStatement();
        if (stmt)
            printf("ORIG %d %s\n", stmt->lineNumber(), tag[stmt->variant()]);

        items = items->getNext();
    }
    
    while (nexts)
    {
        viewGraph(nexts->block);
        nexts = nexts->next;
    }
}

static vector<PrevNode> getPrev(ShadowNode* curr, const set<ShadowNode*>& allNodes)
{
    vector<PrevNode> retVal;

    for (auto& node : allNodes)
        for (auto& next : node->next)
            if (next.shNode == curr)
                retVal.push_back(PrevNode(node, &next.isBackWard, &next.hasRealigns));

    return retVal;
}

static vector<PrevNode> getPrev(ShadowNode* curr, const map<string, vector<FuncInfo*>>& allFuncs)
{
    vector<PrevNode> retVal;

    for (auto& funcByFile : allFuncs)
        for (auto& currF : funcByFile.second)
            for (auto& node : currF->allShadowNodes)
                    for (auto& next : node.second->next)
                        if (next.shNode == curr)
                            retVal.push_back(PrevNode(node.second, &next.isBackWard, &next.hasRealigns));
    
    return retVal;
}

static bool isMoveValid(ShadowNode* moveTo, DIST::Array* array, const set<ShadowNode*>& allShadowNodes)
{
    if (array->GetLocation().first == DIST::l_MODULE)
    {
        auto func = moveTo->location.first->funcPointer;

        bool checkOk = true;
        try {
            array->GetNameInLocationS(moveTo->location.first->funcPointer);
        }
        catch (...) {
            checkOk = false;
        }

        if (!checkOk)
            return false;
    }

    //check added
    for (auto& elem : moveTo->newShadows)
        if (elem.first == array)
            return true;

    set<ShadowNode*> touched = { moveTo };
    set<ShadowNode*> next = { moveTo };

    while (next.size())
    {
        set<ShadowNode*> newNext;

        for (auto& node : next)
        {
            touched.insert(node);

            bool skip = false;
            for (auto& elem : node->shadows)
                if (elem.first == array)
                    skip = true;
            if (skip)
                continue;

            for (auto& edge : node->next)
            {
                if (edge.writeTo.find(array) != edge.writeTo.end())
                    return false;

                if (touched.find(edge.shNode) == touched.end() &&
                    next.find(edge.shNode) == next.end())
                {
                    newNext.insert(edge.shNode);
                }
            }
        }
        next = newNext;
    }
    return true;
}

extern int keepFiles;
static int groupingShadowNodes(set<ShadowNode*>& allShadowNodes, const map<string, vector<FuncInfo*>>& allFuncs,
                               const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls)
{    
    int moveCount = 0;
    bool changes = true;
    while (changes)
    {
        //printf("iter\n");
        changes = false;
        for (auto& shadow : allShadowNodes)
        {
            ShadowNode* currNode = shadow;
            if (currNode->type != PARALLEL_DIR)
                continue;

            vector<set<DIST::Array*>> writes;
            auto prevForCurr = getPrev(currNode, allFuncs);
            for (auto& prev : prevForCurr)
            {
                for (int i = 0; i < prev.shNode->next.size(); ++i)
                {
                    if (prev.shNode->next[i].shNode == currNode)
                    {
                        writes.push_back(prev.shNode->next[i].writeTo);
                        break;
                    }
                }
            }
            if (writes.size() != prevForCurr.size())
                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

            map<DIST::Array*, vector<ShadowElement>> rewriteSh;
            for (auto& sh : currNode->newShadows)
            {
                set<DIST::Array*> allRefs;
                getRealArrayRefs(sh.first, sh.first, allRefs, arrayLinksByFuncCalls);

                bool isMoved = false;
                for (int z = 0; z < prevForCurr.size(); ++z)
                {
                    auto& prev = prevForCurr[z];

                    //check writes to arrays from shadow
                    bool found = false;
                    for (auto& array : writes[z])
                        if (allRefs.find(array) != allRefs.end())
                            found = true;

                    //check writes to all arrays from init shadow
                    if (currNode->shadows.find(sh.first) != currNode->shadows.end())                    
                    {
                        for (int k = 0; k < writes.size() && !found; ++k)
                            for (auto& array : writes[k])
                                if (allRefs.find(array) != allRefs.end())
                                    found = true;
                    }

                    if (found)
                        continue;
                    //dont move back
                    if ((prev.isBackWard && *prev.isBackWard) ||
                        (prev.hasRealigns && *prev.hasRealigns) ||
                        (prev.shNode->type == START) ||
                        sh.first->GetLocation().first == DIST::l_PARAMETER && currNode->location.first != prev.shNode->location.first)
                    {
                        continue;
                    }

                    if (isMoveValid(prev.shNode, sh.first, allShadowNodes))
                    {
                        isMoved = true;
                        prev.shNode->MoveShadow(sh);
                        //printf("move from %d to %d %s\n", currNode->location.second, prev.shNode->location.second, sh.first->GetName().c_str());

                        changes = true;
                        ++moveCount;
                    }
                }

                if (!isMoved)
                {
                    //printf(" **add to re ELSE (%d)\n", currNode->location.second);
                    rewriteSh[sh.first] = sh.second;
                }
            }
            currNode->newShadows = rewriteSh;
            //if (rewriteSh.size())
            //    printf("set to sh %d - newSh %d\n", currNode->location.second, rewriteSh.size());
        }

        /*for (auto& sh : allShadowNodes)
        {
            if (sh->newShadows.size())
                printf("sh %d has %d newSh\n", sh->location.second, sh->newShadows.size());
        }*/
    }

    if (moveCount)
        for (auto& shadow : allShadowNodes)
            shadow->MergeNewShadowElements();

    return moveCount;
}

static bool checkCycleInGraph(ShadowNode* startFrom, const map<ShadowNode*, int>& shadowNum)
{
    bool isCycle = false;
    set<ShadowNode*> q = { startFrom };
    bool change = true;
    while (change)
    {
        change = false;
        

        for (auto& sh : q)
        {
            for (auto& next : sh->next)
            {
                auto nextNode = next.shNode;
                if (nextNode == startFrom)
                {
                    isCycle = true;
                    break;
                }

                if (q.find(nextNode) == q.end())
                {
                    auto it = shadowNum.find(nextNode);
                    if (it->second == 1 && !next.isBackWard)
                    {
                        q.insert(nextNode);
                        change = true;
                    }
                }
            }
        }
    }

    return isCycle;
}

static void detectBackWard(const map<string, FuncInfo*>& mapF, const set<ShadowNode*>& allNodes)
{
    set<ShadowNode*> done;

    map<ShadowNode*, int> shadowNum;
    int z = 0;
    for (auto& elem : allNodes)
        shadowNum[elem] = 0;

    FuncInfo* curr = NULL;
    for (auto& elem : mapF)
        if (elem.second->isMain)
            curr = elem.second;
    checkNull(curr, convertFileName(__FILE__).c_str(), __LINE__);

    set<ShadowNode*> q = { curr->shadowTreeStart };

    while (done.size() != allNodes.size())
    {
        if (q.size() == 0)
        {
            for (auto& elem : allNodes)
            {
                if (done.find(elem) == done.end())
                {
                    if (getPrev(elem, allNodes).size() == 0 && shadowNum[elem] == 0)
                    {
                        q.insert(elem);
                        break;
                    }
                }
            }
        }

        if (q.size() == 0)
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

        set<ShadowNode*> nextQ;
        for (auto& elem : q)
        {
            shadowNum[elem] = 1;
            done.insert(elem);

            for (auto& next : elem->next)
            {
                auto& nextNode = next.shNode;
                if (shadowNum[nextNode] == 0)
                    nextQ.insert(nextNode);
                else
                {
                    if (checkCycleInGraph(nextNode, shadowNum))
                        next.isBackWard = true;
                }
            }
        }
        q = nextQ;
    }
}

static void replacingShadowNodes(FuncInfo* currF)
{
    for (auto& shadow : currF->allShadowNodes)
    {
        SgStatement* parallelDir = (SgStatement*)shadow.first;
        if (shadow.second->newShadows != shadow.second->shadows &&
            parallelDir->variant() == DVM_PARALLEL_ON_DIR)
        {
            SgExpression* dirExp = parallelDir->expr(1);
            if (shadow.second->shadows.size() != 0 && shadow.second->newShadows.size() == 0) //remove shadows
            {
                vector<SgExpression*> newList;
                while (dirExp)
                {
                    if (dirExp->lhs()->variant() != SHADOW_RENEW_OP)
                        newList.push_back(dirExp->lhs());
                    dirExp = dirExp->rhs();
                }
                parallelDir->setExpression(1, *makeExprList(newList));
            }
            else if (shadow.second->newShadows.size() != 0)
            {
                vector<SgExpression*> shadowList;
                for (auto& currSh : shadow.second->newShadows)
                {
                    DIST::Array* currArray = currSh.first;
                    if (currSh.second.size() == 0)
                        continue;

                    const ShadowElement& currElement = currSh.second[0];
                    SgSymbol* s = (SgSymbol*)currArray->GetNameInLocationS(currF->funcPointer);

                    //TODO: if moved from other file
                    /*auto itTmp = currElement.origNameByProc.find(currF);
                    if (itTmp == currElement.origNameByProc.end())
                        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
                    const string nameInProc = itTmp->second;*/
                    const string nameInProc = s->identifier();

                    SgArrayRefExp* newArrayRef = NULL;
                    if (nameInProc == s->identifier())
                        newArrayRef = new SgArrayRefExp(*s);
                    else
                        newArrayRef = new SgArrayRefExp(*findSymbolOrCreate(current_file, nameInProc));
                    newArrayRef->addAttribute(ARRAY_REF, currArray, sizeof(DIST::Array));

                    auto zeroShifts = currElement.bounds;
                    std::fill(zeroShifts.begin(), zeroShifts.end(), make_pair(0, 0));

                    for (auto& elem : genSubscripts(currElement.bounds, zeroShifts))
                        newArrayRef->addSubscript(*elem);

                    SgExpression* p = NULL;
                    if (currElement.corner)
                    {
                        SgKeywordValExp* tmp1 = new SgKeywordValExp("CORNER");
                        p = new SgExpression(ARRAY_OP, newArrayRef, tmp1, NULL);
                    }
                    else
                        p = newArrayRef;
                    shadowList.push_back(p);
                }

                bool changed = false;
                while (dirExp)
                {
                    if (dirExp->lhs()->variant() == SHADOW_RENEW_OP)
                    {
                        dirExp->lhs()->setLhs(*makeExprList(shadowList));

                        changed = true;
                        break;
                    }
                    dirExp = dirExp->rhs();
                }

                if (!changed)
                {
                    SgExpression* newList = new SgExpression(SHADOW_RENEW_OP);
                    newList->setLhs(*makeExprList(shadowList));

                    SgExpression* dirExp = parallelDir->expr(1);
                    SgExpression* tmp = new SgExpression(EXPR_LIST);
                    tmp->setLhs(newList);
                    tmp->setRhs(dirExp);
                    dirExp = tmp;

                    parallelDir->setExpression(1, *dirExp);
                }
            }
        }
    }
}

static void fillShadowAcrossFromParallel(FuncInfo* currF, ShadowNode* newShNode, SgStatement* st, const DIST::Arrays<int>& allArrays)
{
    vector<pair<pair<string, string>, vector<pair<int, int>>>> shadows;
    set<string> corners;

    fillShadowAcrossFromParallel(SHADOW_RENEW_OP, new Statement(st), shadows, corners);

    for (auto& shadow : shadows)
    {
        DIST::Array* array = allArrays.GetArrayByName(shadow.first.second);
        
        string nameInFunc = shadow.first.first;
        auto isCorner = (corners.find(nameInFunc) != corners.end());
        newShNode->shadows[array].push_back(ShadowElement(shadow.second, make_pair(currF, nameInFunc), isCorner));
    }
}

static string nameWithContains(SgStatement* where, SgSymbol* s, const map<string, map<int, set<string>>>& mapCallsF)
{
    if (isSgProgHedrStmt(where))
        return isSgProgHedrStmt(where)->nameWithContains();

    const char* callName = s->identifier();
    auto itF = mapCallsF.find(where->fileName());
    if (itF == mapCallsF.end())
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    auto itL = itF->second.find(where->lineNumber());
    if (itL == itF->second.end())
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    for (auto& call : itL->second)
        if (call.find(callName) != string::npos)
            return call;

    if (IS_BY_USE(s))
    {
        // check in module use
        const map<string, set<SgSymbol*>> byUseInFunc = moduleRefsByUseInFunction(where);

        for (auto& byUse : byUseInFunc)
        {
            bool contains = false;
            for (auto& alt : byUse.second)
                if (alt->identifier() == callName)
                    contains = true;

            if (contains)
            {
                callName = byUse.first.c_str();
                break;
            }
        }

        for (auto& call : itL->second)
            if (call.find(callName) != string::npos)
                return call;
    }

    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    return "";
}

static pair<string, string> getNodeName(ShadowNode* node, const map<FuncInfo*, int>& keyOfFunc, const map<string, map<int, set<string>>>& mapCallsF)
{
    int var = -1;
    string symb = "";
    if (node->type != FUNCTION_CALL)
    {
        auto st = ((SgStatement*)node->info);
        if (!st->switchToFile())
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
        var = st->variant();
        if (st->symbol())
            symb = nameWithContains(st, st->symbol(), mapCallsF);
    }
    else
    {
        auto ex = ((SgExpression*)node->info);
        if (SgFile::switchToFile(node->location.first->fileName) == -1)
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
        var = ex->variant();
        if (ex->symbol())
            symb = nameWithContains(SgStatement::getStatmentByExpression(ex), ex->symbol(), mapCallsF);
    }

    string ret = "F" + std::to_string(keyOfFunc.find(node->location.first)->second);
    if (node->type == START)
        ret += "_START";
    else if (node->type == PROCEDURE_CALL)
        ret += string("_call_") + symb;
    else if (node->type == FUNCTION_CALL)
        ret += string("_func_call_") + symb;
    else if (node->type == STOP)
        ret += "_STOP";
    else if (node->type == END)
        ret += "_END";
    else if (node->type != PARALLEL_DIR)
        ret += "_ERROR";

    ret += "_" + std::to_string(node->location.second);
    
    string color = "";
    if (node->shadows.size())
    {
        color += " [color=\"green\"]";

        vector<string> sh;
        for (auto& elem : node->shadows)
            sh.push_back(elem.first->GetShortName());

        for (int z = 0; z < sh.size(); ++z)
        {
            if (z == 0)
                ret += " (";
            ret += sh[z];
            if (z != sh.size() - 1)
                ret += ",";
            else
                ret += ")";
        }
    }
    return make_pair(ret, color);
}

static string getArrays(const NextNode& next)
{
    string ret = "(";
    for (auto& elem : next.writeTo)
        ret += elem->GetShortName() + ",";
    if (ret[ret.size() - 1] == ',')
        ret[ret.size() - 1] = ')';
    else
        ret += ")";
    return ret == "()" ? "" : ret;
}

static string buildGraphVizNext(FuncInfo* currF, const map<FuncInfo*, int>& keyOfFunc, set<pair<string, string>>& nodes, const map<string, map<int, set<string>>>& mapCallsF)
{
    string graph = "";

    vector<string> lines;
    for (auto& elem : currF->allShadowNodes)
    {
        if (elem.second->location.first != currF)
            continue;

        pair<string, string> curr = getNodeName(elem.second, keyOfFunc, mapCallsF);
        nodes.insert(curr);
        for (auto& next : elem.second->next)
        {
            pair<string, string> nextNode = getNodeName(next.shNode, keyOfFunc, mapCallsF);
            if (next.shNode->location.first == elem.second->location.first)
                nodes.insert(nextNode);
            string line = "\"" + curr.first + "\"->\"" + getNodeName(next.shNode, keyOfFunc, mapCallsF).first + "\"";
            string label = getArrays(next);

            line += "[";
            if (next.isBackWard)
                line += "color=red,penwidth=3.0,";
            if (next.hasRealigns)
                line += "style=dotted,";
            line += "label=\"" + label + "\"]";

            line += "\n";
            lines.push_back(line);
        }
    }

    for (auto& line : lines)
        graph += line;
    return graph;
}

static string buildGraphVizPrev(FuncInfo* currF, const map<FuncInfo*, int>& keyOfFunc, const map<string, vector<FuncInfo*>>& allFuncs, 
                                const map<string, map<int, set<string>>>& mapCallsF)
{
    string graph = "";

    vector<string> lines;
    for (auto& elem : currF->allShadowNodes)
    {
        if (elem.second->location.first != currF)
            continue;

        string curr = getNodeName(elem.second, keyOfFunc, mapCallsF).first;
        auto prevList = getPrev(elem.second, allFuncs);
        for (auto& prev : prevList)
        {
            string nextPrev = getNodeName(prev.shNode, keyOfFunc, mapCallsF).first;
            string line = "\"" + curr + "\"->" + "\"" + getNodeName(prev.shNode, keyOfFunc, mapCallsF).first + "\"";
            line += "[color=green,penwidth=0.5]\n";
            lines.push_back(line);
        }
    }

    for (auto& line : lines)
        graph += line;
    return graph;
}

static void buildGraphViz(const map<string, vector<FuncInfo*>>& allFuncs, const map<string, map<int, set<string>>>& mapCallsF, 
                          string fileOut = "", bool withPrev = false)
{
    string graph = "digraph G{\n";
    map<FuncInfo*, int> keyOfFunc;
    int key = 0;
    for (auto& funcByFile : allFuncs)
        for (auto& currF : funcByFile.second)
            keyOfFunc[currF] = key++;

    for (auto& funcByFile : allFuncs)
    {
        for (auto& currF : funcByFile.second)
        {
            set<pair<string, string>> nodes;
            string lines = buildGraphVizNext(currF, keyOfFunc, nodes, mapCallsF);
            if (nodes.size())
            {
                graph += lines;
                graph += "subgraph \"cluster_" + currF->funcName + ":" + currF->fileName + "\"{\n";
                for (auto& node : nodes)
                    graph += "\"" + node.first + "\"" + node.second + "\n";
                graph += "label=\"" + currF->funcName + ":" + currF->fileName + "\"\n";
                graph += "}\n";
            }
            if (withPrev)
                graph += buildGraphVizPrev(currF, keyOfFunc, allFuncs, mapCallsF);

        }
    }
    graph += "}\n";

    if (fileOut == "out")
        printf("%s\n", graph.c_str());
    else
    {
        FILE* f = NULL;

        if (fileOut == "")
            f = fopen("_shadowNodes.txt", "w");
        else
            f = fopen(fileOut.c_str(), "w");

        if (f == NULL)
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
        fwrite(graph.c_str(), sizeof(char), graph.size(), f);
        fclose(f);
    }

    fflush(NULL);
}

static void removeUnusedEdges(const map<string, vector<FuncInfo*>>& allFuncs)
{
    //remove loops
    for (auto& funcByFile : allFuncs)
    {
        for (auto& currF : funcByFile.second)
        {
            for (auto& elem : currF->allShadowNodes)
            {
                ShadowNode* node = elem.second;
                vector<NextNode> newNext;
                set<DIST::Array*> writesToTheSame;
                bool hasTheSame = false;
                for (auto& next : node->next)
                {
                    if (next.shNode == node)
                    {
                        hasTheSame = true;
                        writesToTheSame.insert(next.writeTo.begin(), next.writeTo.end());
                    }
                }

                for (auto& next : node->next)
                {
                    if (next.shNode != node)
                    {
                        newNext.push_back(next);
                        newNext.back().writeTo.insert(writesToTheSame.begin(), writesToTheSame.end());
                    }
                }

                if (hasTheSame)
                {
                    node->next = newNext;
                    /*vector<PrevNode> newPrev;
                    for (auto& prev : node->prev)
                        if (prev.shNode != node)
                            newPrev.push_back(prev);

                    if (node->prev.size() == newPrev.size())
                        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
                    node->prev = newPrev;*/
                }
            }
        }
    }

    //remove same edges
    bool changed = false;
    for (auto& funcByFile : allFuncs)
    {
        for (auto& currF : funcByFile.second)
        {
            for (auto& elem : currF->allShadowNodes)
            {
                ShadowNode* node = elem.second;

                map<ShadowNode*, NextNode> newNext;
                for (auto& next : node->next)
                {
                    auto it = newNext.find(next.shNode);
                    if (it == newNext.end())
                        newNext[next.shNode] = next;
                    else // TODO: check this - may be unite realigns?
                        it->second.writeTo.insert(next.writeTo.begin(), next.writeTo.end());
                }

                if (newNext.size() != node->next.size())
                {
                    changed = true;
                    node->next.clear();
                    for (auto& elem : newNext)
                        node->next.push_back(elem.second);
                }
            }
        }
    }
}

static map<void*, ShadowNode*> allocated;
void clearAllocatedShadowNodes()
{
    for (auto& elem : allocated)
        delete elem.second;
    allocated.clear();
}

static void removeNode(ShadowNode* node, const map<string, vector<FuncInfo*>>& allFuncs)
{
    for (auto& prev : getPrev(node, allFuncs))
    {
        for (auto& needed : prev.shNode->next)
        {
            if (needed.shNode == node)
            {
                vector<NextNode> newEdges;
                for (auto& next : node->next)
                {
                    if (next.shNode == node)
                        continue;

                    NextNode tmp = needed;
                    tmp.isBackWard |= next.isBackWard;
                    tmp.hasRealigns |= next.hasRealigns;
                    tmp.writeTo.insert(next.writeTo.begin(), next.writeTo.end());
                    tmp.shNode = next.shNode;

                    newEdges.push_back(tmp);
                }

                if (newEdges.size())
                {
                    needed = newEdges[0];
                    for (int z = 1; z < newEdges.size(); ++z)
                        prev.shNode->next.push_back(newEdges[z]);
                }
                break;
            }
        }
    }
}

static void removeStartEnd(ShadowNode* node, FuncInfo* currF, const map<string, vector<FuncInfo*>>& allFuncs)
{
    removeNode(node, allFuncs);
    for (auto& elem : currF->allShadowNodes)
    {
        if (elem.second == node)
        {
            currF->allShadowNodes.erase(elem.first);
            break;
        }
    }
}

static void removeStartEnd(const map<string, vector<FuncInfo*>>& allFuncs)
{
    set<ShadowNode*> toDel;
    for (auto& funcByFile : allFuncs)
    {
        for (auto& currF : funcByFile.second)
        {
            auto nodeS = currF->shadowTreeStart;
            auto nodeE = currF->shadowTreeEnd;

            if (nodeS && getPrev(nodeS, allFuncs).size())
            {
                removeStartEnd(nodeS, currF, allFuncs);
                toDel.insert(currF->shadowTreeStart);
                currF->shadowTreeStart = NULL;
            }

            if (nodeE && nodeE->next.size())
            {
                removeStartEnd(nodeE, currF, allFuncs);
                toDel.insert(currF->shadowTreeEnd);
                currF->shadowTreeEnd = NULL;
            }
        }
    }

    for (auto& del : toDel)
        delete del;
}

static void removeStopStats(const map<string, vector<FuncInfo*>>& allFuncs)
{
    for (auto& funcByFile : allFuncs)
    {
        for (auto& currF : funcByFile.second)
        {
            map<void*, ShadowNode*> newNodes;
            set<ShadowNode*> toDel;
            for (auto& elem : currF->allShadowNodes)
            {
                ShadowNode* node = elem.second;
                if (node->type == STOP)
                {
                    toDel.insert(node);
                    removeNode(node, allFuncs);
                }
                else
                    newNodes[elem.first] = elem.second;
            }

            if (toDel.size())
                currF->allShadowNodes = newNodes;
        }
    }
}

//TODO: what about realigns?
static bool isFunctionEmpty(FuncInfo* func)
{
    if (func->shadowTreeStart &&
        func->shadowTreeStart->next.size() == 1 &&
        func->shadowTreeStart->next[0].shNode == func->shadowTreeEnd)
        return true;
    else
        return false;
}

static bool isFunctionEmptyWithWrites(FuncInfo* func)
{
    if (func->shadowTreeStart &&
        func->shadowTreeStart->next.size() == 1 &&
        func->shadowTreeStart->next[0].shNode == func->shadowTreeEnd &&
        func->shadowTreeStart->next[0].writeTo.size() == 0)
        return true;
    else
        return false;
}

static void replaceCalls(const map<string, vector<FuncInfo*>>& allFuncs, const map<string, FuncInfo*>& mapF, const map<string, map<int, set<string>>>& mapCallsF)
{
    set<ShadowNode*> toDel;

    for (auto& funcByFile : allFuncs)
    {
        if (SgFile::switchToFile(funcByFile.first) == -1)
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

        for (auto& currF : funcByFile.second)
        {
            map<void*, ShadowNode*> newNodes;
            for (auto& nodePair : currF->allShadowNodes)
            {
                ShadowNode* node = nodePair.second;
                if (node->type == PROCEDURE_CALL || node->type == FUNCTION_CALL)
                {
                    string callName = "no";
                    if (node->type == PROCEDURE_CALL)
                    {
                        auto st = ((SgStatement*)node->info);
                        callName = nameWithContains(st, st->symbol(), mapCallsF);
                    }
                    else
                    {
                        auto ex = ((SgExpression*)node->info);
                        callName = nameWithContains(SgStatement::getStatmentByExpression(ex), ex->symbol(), mapCallsF);
                    }

                    auto it = mapF.find(callName);
                    if (it != mapF.end() && it->second->shadowTreeStart && it->second->shadowTreeEnd)
                    {
                        if (!isFunctionEmpty(it->second))
                        {
                            //for (auto& prev : node->prev)
                            //    it->second->shadowTreeStart->prev.push_back(prev);
                            for (auto& prev : getPrev(node, allFuncs))
                                for (auto& next : prev.shNode->next)
                                    if (next.shNode == node)
                                        next.shNode = it->second->shadowTreeStart;
                            //node->prev.clear();

                            for (auto& realNext : node->next)
                                it->second->shadowTreeEnd->next.push_back(realNext);
                            /*for (auto& realNext : node->next)
                                for (auto& prev : realNext.shNode->prev)
                                    if (prev.shNode == node)
                                        prev.shNode = it->second->shadowTreeEnd;*/
                            node->next.clear();
                        }
                        else
                        {
                            for (auto& elem : node->next)
                                elem.writeTo.insert(it->second->shadowTreeStart->next[0].writeTo.begin(), it->second->shadowTreeStart->next[0].writeTo.end());
                            removeNode(node, allFuncs);
                            node->next.clear();
                        }
                        toDel.insert(node);
                    }
                    else
                        newNodes.insert(nodePair);
                }
                else
                    newNodes.insert(nodePair);
            }
            currF->allShadowNodes = newNodes;
        }
    }

    for (auto& del : toDel)
        if (del->type == START || del->type == END)
            delete del;
}



static ShadowNode* allocateNode(SgStatement *st,
                                const vector<bool>& reaching, const map<int, int>& numMap,
                                const map<FuncInfo*, vector<SAPFOR::BasicBlock*>>& cfg,
                                void* info, nodeType type, FuncInfo* func, int line,
                                map<void*, ShadowNode*>& addTo)
{
    ShadowNode* added = NULL;
    auto instBlock = getInstructionAndBlockByStatement(cfg, st);
    if (reaching[numMap.at(instBlock.second->getNumber())])
    {
        auto it = allocated.find(info);
        if (it == allocated.end())
            it = allocated.insert(it, make_pair(info, new ShadowNode(info, type, func, line)));
        added = addTo[info] = it->second;
    }
    return added;
}

static void findFuncCalls(SgStatement* currSt, SgExpression* ex, FuncInfo* currF, const map<string, FuncInfo*>& mapF, const int line, 
                          const map<string, map<int, set<string>>>& mapCallsF,
                          const vector<bool>& reaching, const map<int, int>& numMap,
                          const map<FuncInfo*, vector<SAPFOR::BasicBlock*>>& cfg)
{
    if (ex)
    {
        if (ex->variant() == FUNC_CALL)
        {
            if (isIntrinsicFunctionName(ex->symbol()->identifier()) == false)
            {
                auto it = mapF.find(nameWithContains(currSt, ex->symbol(), mapCallsF));
                if (it != mapF.end() && !isFunctionEmptyWithWrites(it->second))
                {
                    if (currF->allShadowNodes.find(ex) != currF->allShadowNodes.end())
                        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
                    else
                        allocateNode(currSt, reaching, numMap, cfg, ex, FUNCTION_CALL, currF, line, currF->allShadowNodes);
                }
            }
        }

        findFuncCalls(currSt, ex->lhs(), currF, mapF, line, mapCallsF, reaching, numMap, cfg);
        findFuncCalls(currSt, ex->rhs(), currF, mapF, line, mapCallsF, reaching, numMap, cfg);
    }
}

static bool intersectAndAdd(map<ShadowNode*, set<DIST::Array*>>& s1, const map<ShadowNode*, set<DIST::Array*>>& s2)
{
    bool retVal = false;
    
    for (auto& map2 : s2)
    {
        if (map2.first) 
        {
            if (s1.find(map2.first) == s1.end())
            {
                s1[map2.first] = map2.second;
                retVal = true;
            }
            else
            {
                bool res = intersectAndAdd(s1.at(map2.first), map2.second);
                retVal = retVal || res;
            }
        }
    }

    if (s2.find(NULL) != s2.end()) // add to all
    {
        for (auto& map1 : s1)
        {
            if (map1.first)
            {
                bool res = intersectAndAdd(map1.second, s2.at(NULL));
                retVal = retVal || res;
            }
        }
    }

    return retVal;
}

static void fillInitial(map<SAPFOR::BasicBlock*, map<ShadowNode*, set<DIST::Array*>>>& OUT_nodes,
                        map<SAPFOR::BasicBlock*, bool>& OUT_realign,
                        const vector<SAPFOR::BasicBlock*>& blocks, 
                        const map<void*, ShadowNode*>& allShadowNodes,
                        const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls)
{
    for (auto& block : blocks)
    {
        auto& instrs = block->getInstructions();
        bool specific = false;
        for (auto& ir : instrs)
        {
            const auto oper = ir->getInstruction()->getOperation();
            SgStatement* st = ir->getInstruction()->getOperator();

            if ((oper == SAPFOR::CFG_OP::DVM_DIR && st->variant() == DVM_PARALLEL_ON_DIR) ||
                oper == SAPFOR::CFG_OP::EXIT ||
                oper == SAPFOR::CFG_OP::F_CALL)
            {
                auto it = allShadowNodes.find(st);
                if (it == allShadowNodes.end())
                {
                    if (oper == SAPFOR::CFG_OP::F_CALL)
                        it = allShadowNodes.find(ir->getInstruction()->getExpression());
                    else
                        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
                }

                if (it != allShadowNodes.end())
                    OUT_nodes[block][it->second] = set<DIST::Array*>();
                specific = true;
            }
            else if (oper == SAPFOR::CFG_OP::DVM_DIR &&
                (st->variant() == DVM_REALIGN_DIR || st->variant() == DVM_REDISTRIBUTE_DIR))
            {
                OUT_realign[block] = true;
                specific = true;
            }
        }

        if (!specific)
        {
            for (auto& ir : instrs)
            {
                auto instr = ir->getInstruction();
                if (instr->getOperation() == SAPFOR::CFG_OP::STORE)
                {
                    SgStatement* st = instr->getOperator();
                    checkNull(st, convertFileName(__FILE__).c_str(), __LINE__);

                    if (st->variant() == ASSIGN_STAT)
                    {
                        if (isArrayRef(st->expr(0)))
                        {
                            auto s = OriginalSymbol(st->expr(0)->symbol());
                            DIST::Array* array = getArrayFromDeclarated(declaratedInStmt(s), s->identifier());
                            if (array && !array->IsNotDistribute())
                            {
                                set<DIST::Array*> toAdd;
                                addRealArraysRef(toAdd, array, arrayLinksByFuncCalls);
                                OUT_nodes[block][(ShadowNode*)NULL].insert(toAdd.begin(), toAdd.end());
                            }
                        }
                    }
                }
            }
        }
    }
}

static void aggregateInformation(map<SAPFOR::BasicBlock*, map<ShadowNode*, set<DIST::Array*>>>& OUT_nodes,
                                 map<SAPFOR::BasicBlock*, bool>& OUT_realign,
                                 map<SAPFOR::BasicBlock*, map<ShadowNode*, set<DIST::Array*>>>& IN_nodes,
                                 map<SAPFOR::BasicBlock*, bool>& IN_realign,
                                 const vector<SAPFOR::BasicBlock*>& blocks,
                                 const map<void*, ShadowNode*>& allShadowNodes)
{
    bool changed = true;
    //aggregate information
    while (changed)
    {
        changed = false;
        for (auto& block : blocks)
        {
            auto& instrs = block->getInstructions();
            bool specific = false;
            for (auto& ir : instrs)
            {
                const auto oper = ir->getInstruction()->getOperation();
                SgStatement* st = ir->getInstruction()->getOperator();
                SgExpression* ex = ir->getInstruction()->getExpression();
                bool call_spec = false;

                if (oper == SAPFOR::CFG_OP::F_CALL)
                {
                    if (allShadowNodes.find(st) != allShadowNodes.end() ||
                        allShadowNodes.find(ex) != allShadowNodes.end())
                        call_spec = true;
                }


                if ((oper == SAPFOR::CFG_OP::DVM_DIR && st->variant() == DVM_PARALLEL_ON_DIR) ||
                    oper == SAPFOR::CFG_OP::EXIT ||
                    oper == SAPFOR::CFG_OP::F_CALL && call_spec)
                {
                    specific = true;
                    for (auto& prev : block->getPrev())
                    {
                        bool res = intersectAndAdd(IN_nodes[block], OUT_nodes[prev]);
                        if (IN_realign[block] == false && OUT_realign[prev])
                        {
                            IN_realign[block] = true;
                            res = true;
                        }
                        changed = changed || res;
                    }
                }
            }

            if (!specific)
            {
                for (auto& prev : block->getPrev())
                {
                    bool res = intersectAndAdd(IN_nodes[block], OUT_nodes[prev]);
                    if (IN_realign[block] == false && OUT_realign[prev])
                    {
                        IN_realign[block] = true;
                        res = true;
                    }
                    changed = changed || res;
                }

                bool res = intersectAndAdd(OUT_nodes[block], IN_nodes[block]);
                if (OUT_realign[block] == false && IN_realign[block])
                {
                    OUT_realign[block] = true;
                    res = true;
                }
                changed = changed || res;
            }
        }
    }
}

void GroupShadow(const map<string, vector<FuncInfo*>>& allFuncs,
                 const map<string, vector<LoopGraph*>>& loops,
                 const DIST::Arrays<int>& allArrays,
                 const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls,
                 const map<string, CommonBlock*>& commonBlocks)
{
    map<string, FuncInfo*> mapF;
    map<string, map<int, set<string>>> mapCallsF;

    for (auto& byFile : allFuncs)
        for (auto& elem : byFile.second)
        {
            mapF[elem->funcName] = elem;
            for (auto& callInfo : elem->callsFromDetailed)
            {
                auto& call = callInfo.detailCallsFrom;
                mapCallsF[elem->fileName][call.second].insert(call.first);
            }
        }

    map<FuncInfo*, set<FuncInfo*>> callDeps;    
    for (auto& byFunc : mapF)
        callDeps[byFunc.second].insert(byFunc.second->callsFromV.begin(), byFunc.second->callsFromV.end());

    vector<set<FuncInfo*>> ssc;
    vector<set<FuncInfo*>> callLvls = groupByCallDependencies(callDeps, ssc);

    bool deb = (debSh == 1);
    bool withPrev = false;

    //building shadow graph
    for (auto& byLvl : callLvls)
    {
        for (auto& currF : byLvl)
        {
            map<int, LoopGraph*> mapLoops;
            auto itL = loops.find(currF->fileName);
            if (itL != loops.end())
                createMapLoopGraph(itL->second, mapLoops);

            if (SgFile::switchToFile(currF->fileName) == -1)
                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
        
            SgStatement* func = currF->funcPointer->GetOriginal();

            auto cfg = buildCFGforCurrentFunc(func, SAPFOR::CFG_Settings(true, false, false, true, false, true, false), commonBlocks, allFuncs);
            if (cfg.size() != 1)
                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
            auto& blocks = cfg.begin()->second;

            //create reaching blocks
            vector<bool> reaching(blocks.size(), false);
            map<int, int> numMap;
            for (int z = 0; z < blocks.size(); ++z)
                numMap[blocks[z]->getNumber()] = z;

            reaching[numMap[blocks[0]->getNumber()]] = true; // first == START
            bool changed = true;
            while (changed)
            {
                changed = false;
                for (auto& block : blocks)
                {
                    if (reaching[numMap[block->getNumber()]])
                        continue;

                    bool prevReached = false;
                    for (auto& prev : block->getPrev())
                    {
                        if (reaching[numMap[prev->getNumber()]])
                        {
                            if (reaching[numMap[block->getNumber()]] == false)
                            {
                                changed = reaching[numMap[block->getNumber()]] = true;
                                break;
                            }
                        }
                    }
                }
            }

            //start node
            if (func->variant() == ENTRY_STAT)
            {
                auto cp = func->controlParent();
                currF->shadowTreeStart = currF->allShadowNodes[cp] = new ShadowNode(cp, START, currF, func->lineNumber());
                func = cp;
            }
            else
                currF->shadowTreeStart = currF->allShadowNodes[func] = new ShadowNode(func, START, currF, func->lineNumber());

            //find all NODES
            for (SgStatement* st = func; st != func->lastNodeOfStmt(); st = st->lexNext())
            {
                if (st->variant() == DVM_PARALLEL_ON_DIR)
                {
                    auto newShNode = allocateNode(st, reaching, numMap, cfg, st, PARALLEL_DIR, currF, st->lexNext()->lineNumber(), currF->allShadowNodes);
                    if (newShNode)
                        fillShadowAcrossFromParallel(currF, newShNode, st, allArrays);

                    st = st->lexNext();
                    if (st->variant() != FOR_NODE)
                    {
                        if (st->variant() != ASSIGN_STAT)
                            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
                        else
                        {
                            bool found = false;
                            for (auto& data : getAttributes<SgStatement*, SgStatement*>(st, set<int>{ ASSIGN_STAT }))
                            {
                                if (mapLoops.find(data->lineNumber()) != mapLoops.end())
                                    found = true;
                            }

                            if (!found)
                                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
                        }
                    }
                    st = st->lastNodeOfStmt();
                }
                else if (st->variant() == CONTAINS_STMT)
                    break;
                else if (st->variant() == STOP_STAT)
                    allocateNode(st, reaching, numMap, cfg, st, STOP, currF, st->lineNumber(), currF->allShadowNodes);
                else if (st->variant() == PROC_STAT)
                {
                    if (isIntrinsicFunctionName(st->symbol()->identifier()) == false)
                    {
                        auto it = mapF.find(nameWithContains(st, st->symbol(), mapCallsF));
                        if (it != mapF.end() && !isFunctionEmptyWithWrites(it->second))
                            allocateNode(st, reaching, numMap, cfg, st, PROCEDURE_CALL, currF, st->lineNumber(), currF->allShadowNodes);
                    }
                }
                else
                {
                    for (int z = 0; z < 3; ++z)
                        findFuncCalls(st, st->expr(z), currF, mapF, st->lineNumber(), mapCallsF, reaching, numMap, cfg);
                }
            }

            auto lastNode = func->lastNodeOfStmt(); // end node
            auto natureLast = currF->allShadowNodes[lastNode] = new ShadowNode(lastNode, END, currF, lastNode->lineNumber());
            currF->shadowTreeEnd = natureLast;

            map<SAPFOR::BasicBlock*, map<ShadowNode*, set<DIST::Array*>>> IN_nodes;
            map<SAPFOR::BasicBlock*, map<ShadowNode*, set<DIST::Array*>>> OUT_nodes;

            map<SAPFOR::BasicBlock*, bool> IN_realign;
            map<SAPFOR::BasicBlock*, bool> OUT_realign;

            if (blocks.size())
                IN_nodes[blocks[0]][currF->shadowTreeStart] = { };

            fillInitial(OUT_nodes, OUT_realign, blocks, currF->allShadowNodes, arrayLinksByFuncCalls);

            aggregateInformation(OUT_nodes, OUT_realign, IN_nodes, IN_realign, blocks, currF->allShadowNodes);

            // check correctness
            set<ShadowNode*> added;
            for (auto& block : IN_nodes)
                for (auto& pair : block.second)
                    if (pair.first)
                        added.insert(pair.first);
            for (auto& block : OUT_nodes)
                for (auto& pair : block.second)
                    if (pair.first)
                        added.insert(pair.first);

            //with last
            if (added.size() + 1 != currF->allShadowNodes.size())
                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

            //set next
            for (auto& block : blocks)
            {
                auto& instrs = block->getInstructions();
                for (auto& ir : instrs)
                {
                    const auto oper = ir->getInstruction()->getOperation();
                    SgStatement* st = ir->getInstruction()->getOperator();

                    if ((oper == SAPFOR::CFG_OP::DVM_DIR && st->variant() == DVM_PARALLEL_ON_DIR) ||
                        oper == SAPFOR::CFG_OP::EXIT ||
                        oper == SAPFOR::CFG_OP::F_CALL)
                    {
                        auto it = currF->allShadowNodes.find(ir->getInstruction()->getOperator());
                        if (it == currF->allShadowNodes.end())
                        {
                            if (oper == SAPFOR::CFG_OP::F_CALL)
                                it = currF->allShadowNodes.find(ir->getInstruction()->getExpression());
                            else
                                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
                        }

                        if (it != currF->allShadowNodes.end())
                        {
                            auto in_nodes = IN_nodes[block];
                            auto in_realign = IN_realign[block];

                            for (auto& node : in_nodes)
                            {
                                if (node.first == NULL)
                                    continue;

                                node.first->addNext(NextNode(it->second, node.second, in_realign));
                            }
                        }
                    }
                }
            }

            if (blocks.size())
            {
                auto lastPair = getInstructionAndBlockByStatement(cfg, lastNode);

                auto in_nodes = IN_nodes[lastPair.second];
                auto in_realign = IN_realign[lastPair.second];

                for (auto& node : in_nodes)
                {
                    if (node.first == NULL)
                        continue;

                    node.first->addNext(NextNode(natureLast, node.second, in_realign));
                }
            }

            /*if (deb)
                dumpCFG(cfg, false);*/

            deleteCFG(cfg);
            IN_nodes.clear();
            OUT_nodes.clear();
            IN_realign.clear();
            OUT_realign.clear();

            //remove the same nodes
            for (auto& elem : currF->allShadowNodes)
            {
                set<pair<ShadowNode*, set<DIST::Array*>>> tmpl;
                for (auto& listElem : elem.second->next)
                    tmpl.insert(make_pair(listElem.shNode, listElem.writeTo));
                if (tmpl.size() != elem.second->next.size())
                {
                    elem.second->next.clear();
                    for (auto& uniq : tmpl)
                        elem.second->addNext(NextNode(uniq.first, uniq.second));
                }
            }

            //check connectivity of shadow graph
            std::queue<ShadowNode*> q;
            q.push(currF->shadowTreeStart);
            set<ShadowNode*> done;
            while (!q.empty())
            {
                ShadowNode* curr = q.front();
                //for entry
                if (curr->type == END && curr->location.first != currF)
                {
                    q.pop();
                    continue;
                }

                q.pop();
                if (done.find(curr) != done.end())
                    continue;

                done.insert(curr);

                for (auto& next : curr->next)
                    if (done.find(next.shNode) == done.end())
                        q.push(next.shNode);
            }

            if (done.size() != currF->allShadowNodes.size())
            {
                map<string, vector<FuncInfo*>> toPrint;
                toPrint[currF->fileName].push_back(currF);

                if (deb)
                    buildGraphViz(toPrint, mapCallsF, "_sh0.txt", withPrev);
                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
            }
        }
    }

    bool hasShadows = false;
    for (auto& func : mapF)
        for (auto& elem : func.second->allShadowNodes)
            if (elem.second->shadows.size())
                hasShadows = true;

    if (!hasShadows)
        return;

    if (deb)
        buildGraphViz(allFuncs, mapCallsF, "_sh0.txt", withPrev);

    removeStopStats(allFuncs);
    removeUnusedEdges(allFuncs);
    if (deb)
        buildGraphViz(allFuncs, mapCallsF, "_sh1.txt", withPrev);

    replaceCalls(allFuncs, mapF, mapCallsF);
    removeUnusedEdges(allFuncs);
    if (deb)
        buildGraphViz(allFuncs, mapCallsF, "_sh2.txt", withPrev);

    removeStartEnd(allFuncs);
    removeUnusedEdges(allFuncs);
    if (deb)
        buildGraphViz(allFuncs, mapCallsF, "_sh3.txt", withPrev);

    set<ShadowNode*> allNodes;
    for (auto& func : mapF)
        for (auto& elem : func.second->allShadowNodes)
            allNodes.insert(elem.second);

    //detect backward
    detectBackWard(mapF, allNodes);

    if (deb)
        buildGraphViz(allFuncs, mapCallsF, "_sh4.txt", withPrev);

    if (keepFiles)
        buildGraphViz(allFuncs, mapCallsF);

    // grouping
    allNodes.clear();
    for (auto& func : mapF)
        for (auto& elem : func.second->allShadowNodes)
            allNodes.insert(elem.second);

    map<FuncInfo*, int> moveCounts;
    for (auto& shadow : allNodes)
        shadow->newShadows = shadow->shadows;
    int moves = groupingShadowNodes(allNodes, allFuncs, arrayLinksByFuncCalls);
    __spf_print(1, "  shadow total moveCount %d\n", moves);

    if (moves != 0)
    {
        //replacing
        for (auto& funcByFile : allFuncs)
        {
            for (auto& currF : funcByFile.second)
            {
                if (currF->funcPointer->GetOriginal()->switchToFile() == false)
                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
                replacingShadowNodes(currF);
            }
        }
    }

    if (deb)
    {
        for (auto& shadow : allNodes)
            shadow->shadows = shadow->newShadows;
        buildGraphViz(allFuncs, mapCallsF, "_sh5.txt", withPrev);
    }
}