SAPFOR/Sapfor/_src/DvmhRegions/DvmhRegionInserter.cpp

#include "leak_detector.h"

#include <iostream>
#include <set>
#include <map>
#include <vector>
#include <string>
#include <algorithm>
#include <tuple>

#include "DvmhRegionInserter.h"
#include "DvmhRegions/RegionsMerger.h"
#include "../VerificationCode/verifications.h"
#include "../Transformations/function_purifying.h"
#include "../LoopAnalyzer/loop_analyzer.h"
#include "../DirectiveProcessing/directive_parser.h"

using namespace std;


#define DVMH_REG_RD 0
#define DVMH_REG_WT 1

void DvmhRegionInserter::findEdgesForRegions(const vector<LoopGraph*> &loops)
{
    for (auto &loopNode : loops)
    {
        if (!hasLimitsToDvmhParallel(loopNode))
        {
            SgStatement* func_st = getFuncStat(loopNode->loop);
            string fun_name = func_st->symbol()->identifier();
            DvmhRegion *dvmhRegion = new DvmhRegion(loopNode, fun_name);
            // loopNode->inDvmhRegion = true;  // <-- propagation
            regions.push_back(dvmhRegion);
        }
        else if (loopNode->children.size() > 0)
            findEdgesForRegions(loopNode->children);
    }
}

bool DvmhRegionInserter::hasLimitsToDvmhParallel(const LoopGraph *loop) const
{
    bool hasDirective = false;
    if (loop->lineNum > 0 || (loop->lineNum < 0 && loop->altLineNum > 0 && loop->directive))
        hasDirective = (loop->loop->GetOriginal()->lexPrev()->variant() == DVM_PARALLEL_ON_DIR);

    return loop->hasGoto || loop->hasPrints || loop->hasNonPureProcedures || !loop->directive || !hasDirective;
}

void DvmhRegionInserter::insertRegionDirectives()
{
    for (auto &region : regions)
    {
        if (region->getLoops().size() == 0)
            continue;

        SgStatement *regionStartSt = new SgStatement(ACC_REGION_DIR);

        SgStatement *statementBefore = region->getFirstSt()->lexPrev();
        if (!statementBefore || statementBefore->variant() != DVM_PARALLEL_ON_DIR) 
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

        //move comments before
        if (statementBefore->comments())
        {
            regionStartSt->setComments(statementBefore->comments());
            statementBefore->delComments();
        }
        statementBefore->insertStmtBefore(*regionStartSt, *statementBefore->controlParent());

        SgStatement *regionEndSt = new SgStatement(ACC_END_REGION_DIR);
        SgStatement *lastStOfTheLoop = region->getLastSt();
        lastStOfTheLoop->insertStmtAfter(*regionEndSt, *region->getFirstSt()->controlParent());
    }
}

bool DvmhRegionInserter::isLoopParallel(const LoopGraph *loop) const
{
    auto prev_st = loop->loop->lexPrev();

    while (prev_st && isDVM_stat(prev_st))
    {
        if (prev_st->variant() == DVM_PARALLEL_ON_DIR)
            return true;
        prev_st = prev_st->lexPrev();
    }
    return false;
}

void DvmhRegionInserter::parFuncsInNode(LoopGraph *loop, bool isParallel)
{
    // check for parallel
    isParallel |= isLoopParallel(loop);

    // save parallel calls
    if (isParallel)
    {
        for (auto& call : loop->calls)  // mark call as parallel
        {
            auto it = allFunctions.find(call.first);
            if (it == allFunctions.end())
            {
                if (!isIntrinsicFunctionName(call.first.c_str()))
                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
            }
            else
                parallel_functions[it->second].insert(loop);
        }
    }
    else    
        for (auto& nestedLoop : loop->children)
            parFuncsInNode(nestedLoop, isParallel);
}

void DvmhRegionInserter::updateParallelFunctions(const map<string, vector<LoopGraph*>> &loopGraphs)
{    
    for (auto& loopGraph : loopGraphs)
    {
        auto save = current_file->filename();
        if (SgFile::switchToFile(loopGraph.first) == -1)
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
        for (auto& loopNode : loopGraph.second)
        {
            bool isParallel = isLoopParallel(loopNode);
            parFuncsInNode(loopNode, isParallel);
        }

        if (SgFile::switchToFile(save) == -1)
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    }

    bool changes_done = true;
    while (changes_done)
    {
        changes_done = false;
        map<FuncInfo*, set<LoopGraph*>> newList;
        for (auto& funcPair : allFunctions)
        {
            FuncInfo* func = funcPair.second;
            for (auto& callsTo : func->callsTo)
            {
                auto itF = parallel_functions.find(func);

                set<LoopGraph*> added;
                if (itF != parallel_functions.end())
                    added = itF->second;

                auto itTo = parallel_functions.find(callsTo);
                if (itTo != parallel_functions.end())
                {
                    for (auto& loop : itTo->second)
                    {
                        if (added.find(loop) == added.end())
                        {
                            changes_done = true;
                            newList[func].insert(loop);
                        }
                    }                   
                }
            }
        }

        for (auto& newElem : newList)
            for (auto& loop : newElem.second)
                parallel_functions[newElem.first].insert(loop);
    }
}

static SgStatement* skipDvmhRegionInterval(SgStatement *start)
{
    if (start->variant() != ACC_REGION_DIR)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    auto st = start;    
    while (st->variant() != ACC_END_REGION_DIR)
        st = st->lexNext();
    return st->lexNext();
}

ArraySet DvmhRegionInserter::applyUseFilter(const ArraySet& block, const set<DIST::Array*>& filter) const
{
    ArraySet newBlock;
    for (auto& elem : block)
    {
        ArraySet newSet;
        getRealArrayRefs(elem, elem, newSet, arrayLinksByFuncCalls);

        for (auto& orig : newSet)
            if (filter.find(orig) != filter.end())
                newBlock.insert(elem);
    }
    return newBlock;
}

static bool hasLoopsWithDir(LoopGraph* loop)
{
    bool retVal = loop->directive;
    for (auto& ch : loop->children)
        retVal |= hasLoopsWithDir(ch);
    return retVal;
}

static void analyzeFunctionParameter(SgExpression* ex, set<string>& except, ArraySet& arrays, bool isIntrinsicCall)
{
    if (ex)
    {
        if (isArrayRef(ex))
        {
            auto type = ex->symbol()->type();
            if (type->variant() == T_ARRAY)
            {
                auto origS = OriginalSymbol(ex->symbol());
                DIST::Array* currArray = getArrayFromDeclarated(declaratedInStmt(origS), origS->identifier());
                checkNull(currArray, convertFileName(__FILE__).c_str(), __LINE__);

                if (ex->lhs() || ex->rhs())
                    arrays.insert(currArray);
                else if (!ex->lhs() && !ex->rhs())
                {
                    if (isIntrinsicCall)
                        arrays.insert(currArray);
                    else
                        except.insert(ex->symbol()->identifier());
                }

            }
        }
        analyzeFunctionParameter(ex->lhs(), except, arrays, isIntrinsicCall);
        analyzeFunctionParameter(ex->rhs(), except, arrays, isIntrinsicCall);
    }
}

static void analyzeFunctionParameters(SgExpression* paramList, set<string>& except, ArraySet& arrays, bool isIntrinsicCall)
{
    while (paramList)
    {
        SgExpression* ex = paramList->lhs();
        if (ex->variant() == FUNC_CALL)
        {
            bool isIntrinsic = isIntrinsicFunctionName(ex->symbol()->identifier());
            analyzeFunctionParameters(ex->lhs(), except, arrays, isIntrinsic);
        }
        else
            analyzeFunctionParameter(ex, except, arrays, isIntrinsicCall);

        paramList = paramList->rhs();
    }
}

static void createExceptListFromFunctionParameters(SgExpression* ex, set<string>& except, ArraySet& arrays, bool forGetActual)
{
    if (ex)
    {
        if (ex->variant() == FUNC_CALL)
        {
            bool isIntrinsic = isIntrinsicFunctionName(ex->symbol()->identifier()) && forGetActual;
            analyzeFunctionParameters(ex->lhs(), except, arrays, isIntrinsic);
        }
        else
        {
            createExceptListFromFunctionParameters(ex->lhs(), except, arrays, forGetActual);
            createExceptListFromFunctionParameters(ex->rhs(), except, arrays, forGetActual);
        }
    }
}

ArraySet DvmhRegionInserter::excludePrivates(const ArraySet& block) const
{
    ArraySet tmp;
    for (auto& array : block)
    {
        set<DIST::Array*> realArrayRefs;
        getRealArrayRefs(array, array, realArrayRefs, arrayLinksByFuncCalls);

        bool isNotPrivate = true;
        for (auto& elem : realArrayRefs)
            if (elem->IsPrivateInLoop())
                isNotPrivate = false;

        if (!array->IsOmpThreadPrivate() && isNotPrivate)
            tmp.insert(array);
    }
    return tmp;
}

ArraySet DvmhRegionInserter::excludeRemotes(const ArraySet& block, SgStatement* remoteDir) const
{
    checkNull(remoteDir, convertFileName(__FILE__).c_str(), __LINE__);
    if (remoteDir->variant() != DVM_REMOTE_ACCESS_DIR)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    SgStatement* next = remoteDir->lexNext();

     //TODO: record ref!
    if (next->variant() == ASSIGN_STAT)
    {
        auto ex = next->expr(0);
        if (ex->variant() == ARRAY_REF)
            OriginalSymbol(ex->symbol())->identifier();
        else if (ex->variant() == RECORD_REF && ex->rhs()->variant() == ARRAY_REF)
            OriginalSymbol(ex->rhs()->symbol())->identifier();
    }

    set<DIST::Array*> raArrays;

    vector<SgExpression*> remotes;
    for (auto ex = remoteDir->expr(0); ex; ex = ex->rhs())
        fillRemoteFromDir(ex->lhs(), remotes);

    for (auto& remArray : remotes)
    {
        auto arrayR = OriginalSymbol(remArray->symbol());
        SgStatement* decl = declaratedInStmt(arrayR);
        DIST::Array* currArray = getArrayFromDeclarated(decl, arrayR->identifier());
        checkNull(currArray, convertFileName(__FILE__).c_str(), __LINE__);

        set<DIST::Array*> realArrayRefs;
        getRealArrayRefs(currArray, currArray, realArrayRefs, arrayLinksByFuncCalls);

        raArrays.insert(realArrayRefs.begin(), realArrayRefs.end());
    }
    
    ArraySet tmp;
    for (auto& array : block)
    {
        set<DIST::Array*> realArrayRefs;
        getRealArrayRefs(array, array, realArrayRefs, arrayLinksByFuncCalls);

        bool ok = true;
        for (auto& check : realArrayRefs)
            if (raArrays.find(check) != raArrays.end())
                ok = false;
        if (ok)
            tmp.insert(array);
    }
    return tmp;
}

void DvmhRegionInserter::insertForProcCall(SgStatement* st, bool& skipGetActualIfProcCall, bool& skipActualIfProcCall)
{
    const char* procName = st->symbol()->identifier();
    if (isIntrinsicFunctionName(procName) == 0 || isMpiProgram)
    {
        skipGetActualIfProcCall = skipActualIfProcCall = true;

        ArraySet arraysToGetActual, arraysToActual;
        set<string> exceptSymbs;
        analyzeFunctionParameters(st->expr(0), exceptSymbs, arraysToGetActual, false);

        auto writeArrays = get_used_arrs(st, DVMH_REG_WT);
        writeArrays = excludePrivates(writeArrays);

        for (auto& array : writeArrays)
            if (arraysToGetActual.find(array) != arraysToGetActual.end())
                arraysToActual.insert(array);

        arraysToGetActual = applyUseFilter(arraysToGetActual, usedArraysInParallelLoops);
        arraysToActual = applyUseFilter(arraysToActual, usedArraysInParallelLoops);

        insertActualDirective(st, arraysToGetActual, ACC_GET_ACTUAL_DIR, true, &exceptSymbs);
        insertActualDirective(st->lexNext(), arraysToActual, ACC_ACTUAL_DIR, false, &exceptSymbs);
    }
}

SgStatement* DvmhRegionInserter::processSt(SgStatement *st, const vector<ParallelRegion*>* regs)
{
    if (st == NULL)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    if (st->variant() < 0)
        return st->lastNodeOfStmt()->lexNext();

    // Skip regions
    if (st->variant() == ACC_REGION_DIR)
        return skipDvmhRegionInterval(st);

    bool forBlock = false;
    if (st->variant() == FOR_NODE && st->lineNumber() > 0)
    {
        auto it = loopGraphMap.find(st->lineNumber());
        if (it != loopGraphMap.end())
        {
            LoopGraph* loop = it->second;
            if (hasLoopsWithDir(loop) == false)
            {
                forBlock = true;
                set<string> outCalls;
                for (auto& elem : loop->calls)
                    outCalls.insert(elem.first);

                if (checkOutCalls(outCalls))
                    forBlock = false;
                else if (loop->hasGoto || loop->hasStops)
                    forBlock = false;
                else //TODO
                    ;
            }
        }
    }

    // Actualization before remote dir, parallel loops blocks, loops blocks
    if (st->variant() == DVM_REMOTE_ACCESS_DIR || 
        st->variant() == DVM_PARALLEL_ON_DIR || 
        st->variant() == LOGIF_NODE ||
        forBlock)
    {
        SgStatement* raDir = NULL;
        if (st->variant() == DVM_REMOTE_ACCESS_DIR)
            raDir = st;

        if (st->variant() == LOGIF_NODE && ((SgLogIfStmt*)st)->body() &&
            (((SgLogIfStmt*)st)->body()->variant() == DEALLOCATE_STMT ||
             ((SgLogIfStmt*)st)->body()->variant() == ALLOCATE_STMT))
        {
            return st->lexNext();
        }

        SgStatement* block_dir = st;
        while (isDVM_stat(st))
            st = st->lexNext();

        if (st->variant() == LOGIF_NODE && ((SgLogIfStmt*)st)->body() && ((SgLogIfStmt*)st)->body()->variant() == PROC_STAT)
        {
            bool tmp1, tmp2;
            insertForProcCall(((SgLogIfStmt*)st)->body(), tmp1, tmp2);
        }
        else //TODO: use PROC and FUNC analysis
        {
            auto readBlocks = get_used_arrs_for_block(st, DVMH_REG_RD);
            auto writeBlocks = get_used_arrs_for_block(st, DVMH_REG_WT);

            readBlocks = excludePrivates(readBlocks);
            readBlocks = applyUseFilter(readBlocks, writesToArraysInParallelLoops);
            //TODO: need to exclude remotes when the analysis will be clarified
            /*if (raDir)
                readBlocks = excludeRemotes(readBlocks, raDir);*/
            
            writeBlocks = excludePrivates(writeBlocks);
            writeBlocks = applyUseFilter(writeBlocks, writesToArraysInParallelLoops);

            ArraySet unite;
            std::merge(readBlocks.begin(), readBlocks.end(), writeBlocks.begin(), writeBlocks.end(), std::inserter(unite, unite.end()));

            if (!(block_dir->variant() == DVM_REMOTE_ACCESS_DIR && st->variant() == ASSIGN_STAT && readBlocks.size() == 0))
                insertActualDirective(block_dir, unite, ACC_GET_ACTUAL_DIR, true);

            writeBlocks = get_used_arrs_for_block(st, DVMH_REG_WT);
            writeBlocks = excludePrivates(writeBlocks);
            writeBlocks = applyUseFilter(writeBlocks, usedArraysInParallelLoops);

            insertActualDirective(st->lastNodeOfStmt()->lexNext(), writeBlocks, ACC_ACTUAL_DIR, false);
        }
        return st->lastNodeOfStmt()->lexNext();
    }

    // Skip useless
    const int var = st->variant();
    bool skipGetActualIfProcCall = false;
    bool skipActualIfProcCall = false;

    if (var == PROC_STAT)
        insertForProcCall(st, skipGetActualIfProcCall, skipActualIfProcCall);

    if (!isSgExecutableStatement(st) || isDVM_stat(st) ||
        var == ALLOCATE_STMT || var == DEALLOCATE_STMT || 
        st->lastNodeOfStmt() != st || isSPF_stat(st))
    {
        return st->lexNext();
    }

    // Skip operators out of parallel regions 
    if (regs && getAllRegionsByLine(*regs, st->fileName(), st->lineNumber()).size() == 0)
            return st->lexNext();

    //TODO: read and write !!!
    if (!skipGetActualIfProcCall && var != READ_STAT)
    {
        set<string> exceptSymbsForGetActual;
        ArraySet forGetActual;

        if (var != WRITE_STAT)
            for (int z = 0; z < 3; ++z)
               createExceptListFromFunctionParameters(st->expr(z), exceptSymbsForGetActual, forGetActual, true);
                
        auto readArrays = get_used_arrs(st, DVMH_REG_RD);
        readArrays = excludePrivates(readArrays);
        readArrays.insert(forGetActual.begin(), forGetActual.end());
        //filtering by writes in DVMH regions
        readArrays = applyUseFilter(readArrays, writesToArraysInParallelLoops);

        insertActualDirective(st, readArrays, ACC_GET_ACTUAL_DIR, true, &exceptSymbsForGetActual);
    }

    if (!skipActualIfProcCall && var != WRITE_STAT)
    {
        set<string> exceptSymbsForActual;
        ArraySet forActual;

        if (var != READ_STAT)
            for (int z = 0; z < 3; ++z)
                createExceptListFromFunctionParameters(st->expr(z), exceptSymbsForActual, forActual, false);

        auto writeArrays = get_used_arrs(st, DVMH_REG_WT);
        writeArrays = excludePrivates(writeArrays);
        writeArrays.insert(forActual.begin(), forActual.end());
        //filtering by use in DVMH regions
        writeArrays = applyUseFilter(writeArrays, usedArraysInParallelLoops);

        insertActualDirective(st->lexNext(), writeArrays, ACC_ACTUAL_DIR, false, &exceptSymbsForActual);
    }
    return st->lexNext();
}

void DvmhRegionInserter::insertActualDirectives(const vector<ParallelRegion*>* regs)
{
    if (SgFile::switchToFile(file->filename()) == -1)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    __spf_print(1, "  Insert actuals for file %s\n", file->filename());
        
    for (auto& func : funcsForFile)
    {
        // skip parallel funcs
        if (parallel_functions.find(func) != parallel_functions.end())
            continue;

        if (func->doNotAnalyze)
            continue;

        SgStatement* st = func->funcPointer->GetOriginal();
        //skip entry 
        if (st->variant() == ENTRY_STAT)
            continue;
        SgStatement* lastNode = st->lastNodeOfStmt();

        st = st->lexNext();
        while (st != lastNode && st != NULL && st->variant() != CONTAINS_STMT)
            st = processSt(st, regs);

        st = func->funcPointer->GetOriginal();
        while (st != lastNode && st != NULL && st->variant() != CONTAINS_STMT)
        {
            if (st->variant() == ACC_GET_ACTUAL_DIR)
            {
                auto next = st->lexNext();
                if (!next)
                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
                moveLabelBefore(st, next);
            }
            st = st->lexNext();
        }
    }

    if (regs)
    {
        __spf_print(1, "  Insert actuals for arrays copying before and after parallelization areas\n");
        for (auto& area : *regs)
        {
            auto lines = area->GetLines(file->filename());
            if (lines)
            {
                for (auto& regLine : *lines)
                {
                    if (!regLine.isImplicit())
                    {
                        auto forActual = getArrayList(regLine.intervalBefore.first, regLine.intervalBefore.second, true);
                        auto forGetActual = getArrayList(regLine.intervalAfter.first, regLine.intervalAfter.second, false);

                        Statement* forActualSt = regLine.intervalBefore.second;
                        Statement* forGetActualSt = regLine.intervalAfter.first;

                        if (forActual.size())
                        {
                            checkNull(forActualSt, convertFileName(__FILE__).c_str(), __LINE__);
                            auto st = forActualSt->GetOriginal();
                            st->insertStmtBefore(*new SgStatement(ACC_ACTUAL_DIR, NULL, NULL, makeExprList(forActual)), *st->controlParent());
                        }

                        if (forGetActual.size())
                        {
                            checkNull(forGetActualSt, convertFileName(__FILE__).c_str(), __LINE__);
                            auto st = forGetActualSt->GetOriginal();
                            st->insertStmtAfter(*new SgStatement(ACC_GET_ACTUAL_DIR, NULL, NULL, makeExprList(forGetActual)), *st->controlParent());
                        }
                    }
                }
            }
        }
    }
}

vector<SgExpression*> DvmhRegionInserter::getArrayList(Statement* start, Statement* end, bool left) const
{
    vector<SgExpression*> varList;
    if (start && end)
    {
        for (auto st = start->GetOriginal(); st != end->GetOriginal(); st = st->lexNext())
        {
            if (st->variant() == ASSIGN_STAT)
            {
                SgExpression* ref = (left ? st->expr(0) : st->expr(1));
                if (isArrayRef(ref))
                    varList.push_back(ref);
            }
        }
    }
    return varList;
}

void DvmhRegionInserter::insertDirectives(const vector<ParallelRegion*> *regs)
{
    __spf_print(1, "  Find edges for regions\n");
    findEdgesForRegions(loopGraph);

    __spf_print(1, "  Merging regions\n");
    auto merger = RegionsMerger(regions, rw_analyzer);
    regions = merger.mergeRegions();

    for (auto& elem : regions)
    {
        for (auto& loop : elem->getLoops())
        {
            loop->inDvmhRegion = 1;
            loop->propagateDvmhRegion(1);
        }
    }

    __spf_print(1, "  Insert regions\n");
    insertRegionDirectives();
}

static bool hasFuncCalls(SgExpression* ex)
{
    if (ex)
    {
        if (ex->variant() == FUNC_CALL && !isIntrinsicFunctionName(ex->symbol()->identifier()))
            return true;

        bool resH = hasFuncCalls(ex->rhs());
        bool resL = hasFuncCalls(ex->rhs());

        return resH || resL;
    }
    return false;
}

void DvmhRegionInserter::insertActualDirective(SgStatement *st, const ArraySet &arraySet, int variant, bool moveComments, const set<string>* exceptSymbs)
{
    if (!st || (variant != ACC_ACTUAL_DIR && variant != ACC_GET_ACTUAL_DIR) || (arraySet.size() == 0))
        return;

    SgStatement *actualizingSt = new SgStatement(variant);

    vector<SgExpression*> list;
    for (auto &arr : arraySet) 
    {
        string arrayName = arr->GetNameInLocation(st);

        if (exceptSymbs)
            if (exceptSymbs->find(arrayName) != exceptSymbs->end())
                continue;
        list.push_back(new SgVarRefExp(findSymbolOrCreate(file, arrayName)));
    }
    if (list.size() == 0)
        return;
        
    if (variant == ACC_GET_ACTUAL_DIR)
    {
        auto prev = st->lexPrev();
        //filter get_actual list with previuos actual
        if (prev && prev->variant() == ACC_ACTUAL_DIR)
        {
            SgExpression* ex = prev->expr(0);
            set<SgSymbol*> prevActual;
            while (ex)
            {   //only full
                auto ref = ex->lhs();
                if (!ref->lhs() && !ex->rhs())
                    prevActual.insert(ex->lhs()->symbol());
                ex = ex->rhs();
            }

            vector<SgExpression*> listNew;
            for (auto& elem : list)
            {
                if (prevActual.find(elem->symbol()) == prevActual.end())
                    listNew.push_back(elem);
            }
            list = listNew;

            if (list.size() == 0)
                return;
        }
    }
    else if (variant == ACC_ACTUAL_DIR)
    {
        auto prev = st->lexPrev();
        if (prev && prev->variant() == ASSIGN_STAT && !hasFuncCalls(prev->expr(1)) && !hasFuncCalls(prev->expr(0)))
        {
            // actualizing only left part of assign
            list.clear();
            list.push_back(prev->expr(0)->copyPtr());

            //check for ranges
            SgExpression* arrayList = list.back();
            bool needToGetBefore = false;
            if (arrayList)
            {
                SgExpression* listA = arrayList->lhs();
                if (listA && listA->variant() == EXPR_LIST)
                    if (listA->lhs() && listA->lhs()->variant() == DDOT)
                        if (listA->lhs()->lhs() && listA->lhs()->lhs()->variant() == DDOT)
                        {
                            arrayList->setLhs(NULL);
                            needToGetBefore = true;
                        }
            }

            if (needToGetBefore)
            {
                SgStatement* getBefore = new SgStatement(ACC_GET_ACTUAL_DIR);
                getBefore->setExpression(0, makeExprList(list));
                prev->insertStmtBefore(*getBefore, *prev->controlParent());
            }
        }
    }
    actualizingSt->setExpression(0, makeExprList(list));

    st->insertStmtBefore(*actualizingSt, *st->controlParent());
    
    if (moveComments)
    {
        if (st->comments())
        {
            actualizingSt->addComment(st->comments());
            st->delComments();
        }
    }
}

ArraySet DvmhRegionInserter::symbs_to_arrs(set<SgSymbol*> symbols) const
{
    set<DIST::Array*> arrs;

    for (auto& symbol : symbols)
    {
        DIST::Array* arr = getArrayFromDeclarated(declaratedInStmt(symbol), symbol->identifier());
        arrs.insert(arr);
    }
    return arrs;
}

ArraySet DvmhRegionInserter::get_used_arrs(SgStatement* st, int usage_type) const
{
    VarUsages st_usages = rw_analyzer.get_usages(st);
    set<SgSymbol*> st_reads, st_writes;
    if (st_usages.is_undefined())    
        st_reads = st_writes = st_usages.get_all({ VAR_TYPE::VAR_DISTR_ARR, VAR_TYPE::VAR_ARR });
    else 
    {
        st_reads = st_usages.get_reads({ VAR_TYPE::VAR_DISTR_ARR, VAR_TYPE::VAR_ARR });
        st_writes = st_usages.get_writes({ VAR_TYPE::VAR_DISTR_ARR, VAR_TYPE::VAR_ARR });
    }

    if (usage_type == DVMH_REG_RD)
        return symbs_to_arrs(st_reads);
    else if (usage_type == DVMH_REG_WT)
        return symbs_to_arrs(st_writes);
    else
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
}

ArraySet DvmhRegionInserter::get_used_arrs_for_block(SgStatement* st, int usage_type) const
{
    ArraySet usages;
    SgStatement *end = st->lastNodeOfStmt()->lexNext();

    while (st != end && st != NULL)
    {
        // Skip regions
        if (st->variant() == ACC_REGION_DIR)
            st = skipDvmhRegionInterval(st);

        // Skip DVM and SPF dirs
        if (isDVM_stat(st) || isSPF_stat(st))
        {
            st = st->lexNext();
            continue;
        }

        ArraySet st_usages = get_used_arrs(st, usage_type);
        usages.insert(st_usages.begin(), st_usages.end());
        st = st->lexNext();
    }
    return usages;
}

static bool filterFromList(SgStatement* st, const set<string>& idents, bool exclude = false)
{
    bool empty = false;
    SgExpression* list = st->expr(0);
    vector<SgExpression*> newList;

    int total = 0;
    while (list)
    {
        if (exclude)
        {
            if (idents.find(list->lhs()->symbol()->identifier()) == idents.end())
                newList.push_back(list->lhs());
        }
        else
        {
            if (idents.find(list->lhs()->symbol()->identifier()) != idents.end())
                newList.push_back(list->lhs());
        }
        total++;
        list = list->rhs();
    }

    if (newList.size() == 0)
        empty = true;
    else if (total != newList.size())
        st->setExpression(0, makeExprList(newList));

    return empty;
}

static string getInterfaceBlock(SgStatement* func, const FuncParam& pars)
{
    string oldFile = current_file->filename();
    if (!func->switchToFile())
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    auto copy = duplicateProcedure(func, NULL, false, false, false, true);
    const set<string> idents(pars.identificators.begin(), pars.identificators.end());

    //remove all exec
    SgStatement* st = copy->lexNext();
    SgStatement* last = copy->lastNodeOfStmt();
    vector<SgStatement*> toExtract;
    while (st != last)
    {
        if (isDVM_stat(st) || isSPF_stat(st))
        {
            if (st->variant() != ACC_ROUTINE_DIR)
            {
                SgStatement* next = st->lexNext();
                st->extractStmt();
                st = next;
            }
            else
                st = st->lexNext();
        }
        else if (isSgExecutableStatement(st))
        {
            SgStatement* next = st->lastNodeOfStmt();
            if (next != last)
                next = next->lexNext();
            toExtract.push_back(st);
            st = next;
        }
        else
            st = st->lexNext();
    }  

    //remove unused declarations
    st = copy->lexNext();
    while (st != last)
    {
        const int var = st->variant();
        if (var == VAR_DECL
            || var == VAR_DECL_90
            || var == DIM_STAT
            || var == INTENT_STMT
            || var == EXTERN_STAT)
        {
            bool empty = filterFromList(st, idents);
            if (empty)
            {
                SgStatement* next = st->lexNext();
                toExtract.push_back(st);
                st = next;
                continue;
            }
        }
        else if (!isDVM_stat(st) && !isSPF_stat(st))
            toExtract.push_back(st);

        if (st->variant() == CONTAINS_STMT)
            break;
        st = st->lexNext();
    }

    for (auto& elem : toExtract)
        elem->extractStmt();

    string retVal = copy->unparse();

    if (SgFile::switchToFile(oldFile) == -1)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    return retVal;
}

static void insertInterface(SgStatement* func, const string& iface, const string& fName)
{
    string oldFile = current_file->filename();
    if (!func->switchToFile())
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    SgStatement* st = func->lexNext();
    SgStatement* last = func->lastNodeOfStmt();
    while (st != last)
    {
        if (st->variant() == VAR_DECL || st->variant() == VAR_DECL_90)
        {
            bool empty = filterFromList(st, { fName }, true);
            if (empty)
            {
                SgStatement* next = st->lexNext();
                st->extractStmt();
                st = next;
                continue;
            }
        }

        if (isSgExecutableStatement(st))
            break;
        st = st->lexNext();
    }
    SgStatement* ifaceBlock = new SgStatement(INTERFACE_STMT);
    addControlEndToStmt(ifaceBlock->thebif);

    ifaceBlock->setlineNumber(getNextNegativeLineNumber()); // st->lineNumber()
    ifaceBlock->setFileName(st->fileName());
    st->insertStmtBefore(*ifaceBlock, *st->controlParent());    
    ifaceBlock->lastNodeOfStmt()->addComment(iface.c_str());

    if (SgFile::switchToFile(oldFile) == -1)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
}

static LoopGraph* getParallelLoop(LoopGraph* loop)
{
    auto prev_st = loop->loop->lexPrev();

    while (prev_st && isDVM_stat(prev_st))
    {
        if (prev_st->variant() == DVM_PARALLEL_ON_DIR)
            return loop;
        prev_st = prev_st->lexPrev();
        loop = loop->parent;
    }

    return NULL;
}

static set<LoopGraph*> getParallelLoops(const std::set<LoopGraph*>& loops)
{
    set<LoopGraph*> retVal;
    for (auto& elem : loops)
    {
        string oldFile = current_file->filename();
        if (!elem->loop->switchToFile())
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

        auto parLoop = getParallelLoop(elem);
        checkNull(parLoop, convertFileName(__FILE__).c_str(), __LINE__);
        retVal.insert(parLoop);

        if (SgFile::switchToFile(oldFile) == -1)
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    }
    return retVal;
}

static set<DIST::Array*> getPrivateArrays(const set<LoopGraph*>& parLoops,
                                          const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls)
{
    set<DIST::Array*> retVal;

    for (auto& loop : parLoops)
    {
        string oldFile = current_file->filename();
        if (!loop->loop->switchToFile())
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

        for (auto& priv : loop->directive->privates)
        {
            bool isArray = false;
            if (isArrayType(priv->type()))
            {
                auto type = isSgArrayType(priv->type());
                if (type && type->dimension())
                    isArray = true;
            }

            if (!isArray)
                continue;

            DIST::Array* arr = getArrayFromDeclarated(declaratedInStmt(priv), priv->identifier());
            checkNull(arr, convertFileName(__FILE__).c_str(), __LINE__);

            set<DIST::Array*> realArrayRefs;
            getRealArrayRefs(arr, arr, realArrayRefs, arrayLinksByFuncCalls);

            for (auto& elem : realArrayRefs)
                retVal.insert(elem);
        }

        if (SgFile::switchToFile(oldFile) == -1)
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    }

    return retVal;
}

static SgExpression* getPrivateArraysInPar(const FuncInfo* funcInfo, const std::set<LoopGraph*>& inLoops,
                                           const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls,
                                           SgExpression* addedPrivList)
{
    if (!funcInfo)
        return NULL;

    map<string, SgExpression*> addedPriv;
    SgExpression* ex = addedPrivList;
    while (ex)
    {
        if (ex->lhs() && ex->lhs()->variant() == ACC_PRIVATE_OP)
        {
            ex = ex->lhs()->lhs();
            while (ex)
            {
                if (ex->lhs() && ex->lhs()->variant() == VAR_REF)
                    addedPriv[ex->lhs()->symbol()->identifier()] = ex->lhs();
                ex = ex->rhs();
            }
            break;
        }
        ex = ex->rhs();
    }

    SgStatement* func = funcInfo->funcPointer->GetOriginal();
    auto prog = isSgProgHedrStmt(func);
    checkNull(prog, convertFileName(__FILE__).c_str(), __LINE__);
    
    set<LoopGraph*> inParLoops = getParallelLoops(inLoops);
    set<DIST::Array*> privArrays = getPrivateArrays(inLoops, arrayLinksByFuncCalls);

    for (int z = 0; z < prog->numberOfParameters(); ++z)
    {
        SgSymbol* par = prog->parameter(z);
        bool isArray = false;
        if (isArrayType(par->type()))
        {
            auto type = isSgArrayType(par->type());
            if (type && type->dimension())
                isArray = true;
        }

        if (isArray && addedPriv.count(par->identifier()) == 0)
        {
            DIST::Array* arr = getArrayFromDeclarated(declaratedInStmt(par), par->identifier());
            checkNull(arr, convertFileName(__FILE__).c_str(), __LINE__);

            set<DIST::Array*> realArrayRefs;
            getRealArrayRefs(arr, arr, realArrayRefs, arrayLinksByFuncCalls);

            bool found = false;
            for (auto& elem : realArrayRefs)
                if (privArrays.find(elem) != privArrays.end())
                    found = true;

            if (found)
                addedPriv[par->identifier()] = new SgVarRefExp(par);
        }
    }

    if (addedPriv.size())
    {
        vector<SgExpression*> list;
        for (auto& elem : addedPriv)
            list.push_back(elem.second);

        return makeExprList(list, false);
    }
    else
        return NULL;
}

static SgStatement* getInsertionPlace(SgStatement* func)
{
    SgStatement* place = func->lexNext();
    SgStatement* insertAfter = NULL;
    for (auto st = place; st != func->lastNodeOfStmt(); st = st->lexNext())
    {
        if (isSPF_stat(st) || isDVM_stat(st))
            continue;

        if (st->variant() == CONTAINS_STMT)
            break;

        if (isSgExecutableStatement(st))
            break;

        if (st->variant() == IMPL_DECL)
            insertAfter = st;
        else if (st->variant() == USE_STMT)
            insertAfter = st;
    }

    if (insertAfter)
        return insertAfter->lexNext();
    else
        return place;
}

static void insertRoutine(SgStatement* func, const FuncInfo* funcInfo, const std::set<LoopGraph*>& inLoops,
                          const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls)
{
    string oldFile = current_file->filename();
    if (!func->switchToFile())
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    
    SgStatement* st = func->lexNext();
    SgStatement* last = func->lastNodeOfStmt();

    SgStatement* routine = NULL;
    while (st != last)
    {
        if (st->variant() == ACC_ROUTINE_DIR)
        {
            routine = st;
            break;                
        }
        st = st->lexNext();
    }
    
    if (!routine)
    {
        st = getInsertionPlace(func);
        routine = new SgStatement(ACC_ROUTINE_DIR);
        st->insertStmtBefore(*routine, *func);
    }

    SgExpression* list = getPrivateArraysInPar(funcInfo, inLoops, arrayLinksByFuncCalls, routine->expr(0));
    if (list)
    {
        list = new SgExpression(EXPR_LIST, new SgExpression(ACC_PRIVATE_OP, list));
        routine->setExpression(0, list);
    }

    if (SgFile::switchToFile(oldFile) == -1)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
}

static bool isPure(SgStatement* func)
{
    string oldFile = current_file->filename();
    if (!func->switchToFile())
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    bool retVal = ((func->symbol()->attributes() & PURE_BIT) != 0);

    if (SgFile::switchToFile(oldFile) == -1)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    return retVal;
}

void DvmhRegionInserter::createInterfaceBlockForOutCall(FuncInfo* func, FuncInfo* callFrom)
{
    insertInterface(func->funcPointer, getInterfaceBlock(callFrom->funcPointer->GetOriginal(), callFrom->funcParams), callFrom->funcName);
}

void DvmhRegionInserter::createInterfaceBlockForParallelFunctions(bool onlyRoutine)
{
    for (auto& func_pair : parallel_functions)
    {
        const auto& parF = func_pair.first;
        const auto& inLoops = func_pair.second;

        for (auto& callTo : parF->callsTo)
        {
            if (!isPure(parF->funcPointer->GetOriginal()))
                continue;

            if (callTo->fileName != parF->fileName && onlyRoutine)
            {
                insertRoutine(parF->funcPointer->GetOriginal(), parF, inLoops, arrayLinksByFuncCalls);
                continue;
            }

            if (callTo->fileName != parF->fileName || isPure(callTo->funcPointer->GetOriginal()))
            {
                auto it = callTo->interfaceBlocks.find(parF->funcName);
                if (it == callTo->interfaceBlocks.end())
                {
                    callTo->interfaceBlocks[parF->funcName] = NULL;
                    createInterfaceBlockForOutCall(callTo, parF);
                }
                else if (it->second) // interface not inserted as comment
                {
                    SgStatement* st = callTo->funcPointer->GetOriginal();

                    string oldFile = current_file->filename();
                    if (!st->switchToFile())
                        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

                    SgStatement* last = st->lastNodeOfStmt();
                    SgStatement* iface = NULL;

                    st = st->lexNext();
                    while (st != last)
                    {
                        if (st->variant() == FUNC_HEDR || st->variant() == PROC_HEDR)
                        {
                            if (st->controlParent()->variant() == INTERFACE_STMT && 
                                st->symbol()->identifier() == parF->funcName)
                            {
                                iface = st;
                                insertRoutine(iface, parF, inLoops, arrayLinksByFuncCalls);
                                break;
                            }
                        }

                        if (st->variant() == CONTAINS_STMT)
                            break;
                        if (isSgExecutableStatement(st))
                            break;
                        st = st->lexNext();
                    }

                    checkNull(iface, convertFileName(__FILE__).c_str(), __LINE__);

                    if (SgFile::switchToFile(oldFile) == -1)
                        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
                }
            }
        }
    }
}

void DvmhRegionInserter::createInterfaceBlockForOutCalls(FuncInfo* func)
{
    for (auto& callFrom : func->callsFromV)
    {
        if (func->interfaceBlocks.find(callFrom->funcName) == func->interfaceBlocks.end()
            && isPure(callFrom->funcPointer->GetOriginal()))
        {
            func->interfaceBlocks[callFrom->funcName] = callFrom;
            createInterfaceBlockForOutCall(func, callFrom);
        }
    }
}

void DvmhRegionInserter::removePrivatesFromParallelLoops()
{
    if (loopGraph.size() != 0 && loopGraphMap.size() == 0)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    for (auto& loopPair : loopGraphMap)
    {
        auto loop = loopPair.second;
        if (loop->directive && loop->inDvmhRegion <= 0)
        {
            SgStatement* lexPrev = loop->loop->GetOriginal()->lexPrev();

            if (lexPrev->variant() == DVM_PARALLEL_ON_DIR)
            {
                if (sharedMemoryParallelization == 1)
                    lexPrev->deleteStmt();
                else
                {
                    SgExprListExp* list = isSgExprListExp(lexPrev->expr(1));
                    if (list)
                    {
                        vector<SgExpression*> newList;
                        for (SgExpression* ex = list; ex; ex = ex->rhs())
                            if (ex->lhs()->variant() != ACC_PRIVATE_OP)
                                newList.push_back(ex->lhs());

                        lexPrev->setExpression(1, makeExprList(newList));
                    }
                }
            }
        }
    }
}

void DvmhRegionInserter::addPrivatesToParallelLoops()
{
    if (loopGraph.size() != 0 && loopGraphMap.size() == 0)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

    for (auto& loopPair : loopGraphMap)
    {
        auto loop = loopPair.second;

        // last condition means manual parallelization
        if (loop->directive && loop->inDvmhRegion == 1 && loop->directive->parallel.size() == 0)
        {
            SgStatement* lexPrev = loop->loop->GetOriginal()->lexPrev();
            if (lexPrev->variant() == DVM_PARALLEL_ON_DIR)
            {
                auto attrs = getAttributes<SgStatement*, SgStatement*>(loop->loop->GetOriginal(), set<int>{ SPF_ANALYSIS_DIR });
                set<Symbol*> privates;

                for (auto attr : attrs)
                    fillPrivatesFromComment(new Statement(attr), privates);

                if (privates.size())
                {
                    SgExprListExp* list = isSgExprListExp(lexPrev->expr(1));
                    if (list)
                    {
                        bool privatesFound = false;
                        bool changed = false;

                        vector<SgExpression*> newList;
                        for (SgExpression* ex = list; ex; ex = ex->rhs())
                        {
                            if (ex->lhs()->variant() == ACC_PRIVATE_OP)
                            {
                                privatesFound = true;

                                set<SgSymbol*> privatesInDir;
                                set<string> privatesInDirStr;

                                SgExpression* list = ex->lhs()->lhs();
                                while (list)
                                {
                                    privatesInDir.insert(list->lhs()->symbol());
                                    privatesInDirStr.insert(list->lhs()->symbol()->identifier());
                                    list = list->rhs();
                                }

                                for (auto& elem : privates)
                                {
                                    if (privatesInDirStr.find(elem->identifier()) == privatesInDirStr.end())
                                    {
                                        changed = true;
                                        privatesInDirStr.insert(elem->identifier());
                                        privatesInDir.insert(elem->GetOriginal());
                                    }
                                }

                                if (changed)
                                {
                                    vector<SgExpression*> privAccess;
                                    for (auto& elem : privatesInDir)
                                        privAccess.push_back(new SgVarRefExp(elem));
                                    newList.push_back(new SgExpression(ACC_PRIVATE_OP, makeExprList(privAccess), NULL));
                                }
                            }
                            else
                                newList.push_back(ex->lhs());
                        }

                        if (privatesFound == false)
                        {
                            vector<SgExpression*> privAccess;
                            for (auto& elem : privates)
                                privAccess.push_back(new SgVarRefExp(elem->GetOriginal()));

                            newList.push_back(new SgExpression(ACC_PRIVATE_OP, makeExprList(privAccess), NULL));
                            changed = true;
                        }

                        if (changed)
                            lexPrev->setExpression(1, makeExprList(newList));
                    }
                    else
                    {
                        vector<SgExpression*> privAccess;
                        for (auto& elem : privates)
                            privAccess.push_back(new SgVarRefExp(elem->GetOriginal()));

                        lexPrev->setExpression(1, makeExprList(vector<SgExpression*>{ new SgExpression(ACC_PRIVATE_OP, makeExprList(privAccess), NULL)}, false));
                    }
                }
            }
        }
    }
}

void DvmhRegionInserter::addUsedArrays(set<DIST::Array*>& arrays)
{
    for (auto& loopPair : loopGraphMap)
    {
        auto loop = loopPair.second;
        if (loop->inDvmhRegion == 1)
            arrays.insert(loop->usedArraysAll.begin(), loop->usedArraysAll.end());
    }
}

void DvmhRegionInserter::addUsedWriteArrays(set<DIST::Array*>& arrays)
{
    for (auto& loopPair : loopGraphMap)
    {
        auto loop = loopPair.second;
        if (loop->inDvmhRegion == 1)
            arrays.insert(loop->usedArraysWriteAll.begin(), loop->usedArraysWriteAll.end());
    }
}

static set<DIST::Array*> 
    insertDeclare(const set<DIST::Array*>& usedArraysInRegions,
                  const set<DIST::Array*>& usedWriteArraysInRegions,
                  const map<DIST::Array*, set<DIST::Array*>> arrayLinksByFuncCalls,
                  SgStatement* main)
{
    set<DIST::Array*> commonArrays;

    vector<DIST::Array*> usedAll;
    std::set_union(usedArraysInRegions.begin(), usedArraysInRegions.end(),
                   usedWriteArraysInRegions.begin(), usedWriteArraysInRegions.end(),
                   std::back_inserter(usedAll));

    set<DIST::Array*> added;
    map<SgStatement*, set<Symbol*>> toDeclareByFunc;

    for (auto& array : usedAll)
    {
        set<DIST::Array*> realRef;
        getRealArrayRefs(array, array, realRef, arrayLinksByFuncCalls);

        for (auto& realArray : realRef)
        {
            if (added.count(realArray) != 0 || !realArray->IsNotDistribute())
                continue;

            SgStatement* declStat = NULL;

            if (realArray->GetLocation().first != DIST::l_COMMON)
            {
                if (std::count(usedAll.begin(), usedAll.end(), realArray) == 0)
                {
                    auto declInfo = *realArray->GetDeclInfo().begin();
                    declStat = SgStatement::getStatementByFileAndLine(declInfo.first, declInfo.second);
                    checkNull(declStat, convertFileName(__FILE__).c_str(), __LINE__);
                }
            }
            else
            {
                commonArrays.insert(realArray);
                auto decls = realArray->GetDeclInfo();
                for (auto& decl : decls)
                {
                    declStat = SgStatement::getStatementByFileAndLine(decl.first, decl.second);
                    if (declStat == NULL) // check in inlcudes 
                    {                        
                        for (auto st = main; st != main->lastNodeOfStmt() && !declStat; st = st->lexNext())
                        {
                            if (st->fileName() == decl.first && st->lineNumber() == decl.second)
                                declStat = st;
                        }

                        if (declStat)
                            break;
                    }
                    else
                    {
                        declStat = getFuncStat(declStat);
                        if (declStat != main)
                        {
                            declStat = NULL;
                            continue;
                        }
                    }
                }
            }

            if (declStat)
            {
                added.insert(realArray);
                declStat = getFuncStat(declStat);
                checkNull(declStat, convertFileName(__FILE__).c_str(), __LINE__);

                toDeclareByFunc[declStat].insert(realArray->GetDeclSymbol());
            }
        }
    }

    for (auto& declPair : toDeclareByFunc)
    {
        SgStatement* func = declPair.first;
        const set<Symbol*>& symbols = declPair.second;

        if (!func->switchToFile())
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

        set<string> added;
        vector<SgExpression*> list;
        for (auto& s : symbols)
        {
            if (added.count(s->identifier()) == 0)
            {
                added.insert(s->identifier());
                list.push_back(new SgVarRefExp(s));
            }
        }

        auto place = getInsertionPlace(func);
        place->insertStmtBefore(*new SgStatement(ACC_DECLARE_DIR, NULL, NULL, makeExprList(list)), *func);
    }

    return commonArrays;
}

int insertDvmhRegions(SgProject& project, int files, const vector<ParallelRegion*>& parallelRegions,
                      map<string, vector<FuncInfo*>>& allFuncInfo,
                      map<string, vector<LoopGraph*>> loopGraph,
                      ReadWriteAnalyzer& rw_analyzer,
                      map<string, vector<Messages>>& SPF_messages,
                      const map<DIST::Array*, set<DIST::Array*>> arrayLinksByFuncCalls)
{
    int internalExit = 0;

    vector<DvmhRegionInserter*> inserters;
    const bool regionCondition = ((parallelRegions.size() == 0 && parallelRegions[0]->GetName() == "DEFAULT") || sharedMemoryParallelization == 1);

    set<DIST::Array*> usedArraysInRegions;
    set<DIST::Array*> usedWriteArraysInRegions;

    for (int i = files - 1; i >= 0; --i)
    {
        SgFile* file = &(project.file(i));
        __spf_print(1, "  ==> Start region inserter for file %s\n", file->filename());

        map<string, FuncInfo*> mapOfFuncs;
        createMapOfFunc(allFuncInfo, mapOfFuncs);

        auto loopsForFile = getObjectForFileFromMap(file->filename(), loopGraph);
        auto funcsForFile = getObjectForFileFromMap(file->filename(), allFuncInfo);

        for (auto& loop : loopsForFile)
            loop->analyzeParallelDirs();

        DvmhRegionInserter* regionInserter = new DvmhRegionInserter(file, loopsForFile, rw_analyzer, arrayLinksByFuncCalls, mapOfFuncs, funcsForFile, sharedMemoryParallelization == 1);
        inserters.push_back(regionInserter);

        //collect info about <parallel> functions
        regionInserter->updateParallelFunctions(loopGraph);

        if (regionCondition)
            regionInserter->insertDirectives(NULL);
        else
            regionInserter->insertDirectives(&parallelRegions);

        //remove privates from loops out of DVMH region 
        //remove parallel directives from loops out of DVMH region for MPI regime
        regionInserter->removePrivatesFromParallelLoops();

        //add privates to parallel loops with manual parallelization in DVMH regions
        regionInserter->addPrivatesToParallelLoops();

        regionInserter->addUsedArrays(usedArraysInRegions);
        regionInserter->addUsedWriteArrays(usedWriteArraysInRegions);

        setPureStatus(regionInserter->getParallelFunctions());
    }

    for (int i = files - 1, k = 0; i >= 0; --i, ++k)
    {
        SgFile* file = &(project.file(i));
        // insert ROUTINE directive if needed
        inserters[k]->createInterfaceBlockForParallelFunctions();
    }

    for (int i = files - 1, k = 0; i >= 0; --i, ++k)
    {
        SgFile* file = &(project.file(i));

        // create interface for 'parallel' functions
        inserters[k]->createInterfaceBlockForParallelFunctions(false);
    }

    SgStatement* main = findMainUnit(&project, SPF_messages);
    checkNull(main, convertFileName(__FILE__).c_str(), __LINE__);

    set<DIST::Array*> commonArrays = insertDeclare(usedArraysInRegions, usedWriteArraysInRegions, arrayLinksByFuncCalls, main);
    internalExit = checkCommonInMainUnit(project, SPF_messages, commonArrays, false);

    for (auto& regionInserter : inserters)
    {
        regionInserter->updateUsedArrays(usedArraysInRegions, usedWriteArraysInRegions);

        if (regionCondition)
            regionInserter->insertActualDirectives(NULL);
        else
            regionInserter->insertActualDirectives(&parallelRegions);
        delete regionInserter;
    }

    return internalExit;
}