finalyze moving

src/DirectiveProcessing/directive_analyzer.cpp

@@ -0,0 +1,92 @@
+#include "../Utils/leak_detector.h"
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <cstdint>
+
+#include <string>
+#include <fstream>
+#include <iostream>
+
+#include <vector>
+#include <map>
+#include <set>
+#include <utility>
+#include <assert.h>
+
+#include "../Distribution/DvmhDirective.h"
+#include "../GraphLoop/graph_loops.h"
+#include "directive_analyzer.h"
+#include "../Utils/utils.h"
+
+using std::vector;
+using std::map;
+
+static void recUniteAcrossOut(LoopGraph *currLoop)
+{
+    if (currLoop->perfectLoop > 1)
+    {
+        recUniteAcrossOut(currLoop->children[0]);
+        if (currLoop->directive)
+            currLoop->acrossOutAttribute.insert(currLoop->children[0]->acrossOutAttribute.begin(), currLoop->children[0]->acrossOutAttribute.end());
+    }
+}
+
+static void recUnite(vector<LoopGraph*> &loopGraph)
+{
+    for (int i = 0; i < loopGraph.size(); ++i)
+    {
+        LoopGraph &currLoop = *loopGraph[i];
+        if (currLoop.perfectLoop > 1)
+        {
+            vector<ParallelDirective*> united;
+            vector<ParallelDirective*> elems;
+
+            LoopGraph *elem = loopGraph[i];
+            for (int k = 0; k < currLoop.perfectLoop; ++k)
+            {
+                elems.push_back(elem->directive);
+                if (k != currLoop.perfectLoop - 1)
+                    elem = elem->children[0];
+            }
+
+            united.push_back(elems.back());
+            for (int k = currLoop.perfectLoop - 2; k >= 0; --k)
+            {
+                ParallelDirective *first = elems[k];
+                ParallelDirective *second = united.back();
+                ParallelDirective *newDir = NULL;
+                if (first != NULL && second != NULL)
+                    newDir = *first + *second;
+                else if (first != NULL)
+                    newDir = new ParallelDirective(*first);
+                else if (first == NULL && second != NULL)
+                {
+                    newDir = new ParallelDirective(*second);
+                    newDir->parallel.insert(newDir->parallel.begin(), "*");
+                }
+
+                united.push_back(newDir);
+            }
+
+            elem = loopGraph[i];
+            for (int k = 0; k < currLoop.perfectLoop - 1; ++k)
+            {
+                if (elem->directive)
+                    delete elem->directive;
+
+                elem->directive = united[currLoop.perfectLoop - k - 1];
+                elem = elem->children[0];
+            }
+            recUniteAcrossOut(loopGraph[i]);
+        }
+        else
+            recUnite(currLoop.children);
+    }
+}
+
+void UniteNestedDirectives(vector<LoopGraph*> &loopGraph)
+{
+    recUnite(loopGraph);
+}

src/DirectiveProcessing/directive_analyzer.h

@@ -0,0 +1,6 @@
+#pragma once
+
+#include <vector>
+#include "../GraphLoop/graph_loops.h"
+
+void UniteNestedDirectives(std::vector<LoopGraph*> &loopGraph);

src/DirectiveProcessing/directive_creator.cpp

@@ -0,0 +1,490 @@
+#include "../Utils/leak_detector.h"
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <cstdint>
+#include <assert.h>
+
+#include "dvm.h"
+
+#include "../ParallelizationRegions/ParRegions_func.h"
+
+#include "../Distribution/GraphCSR.h"
+#include "../Distribution/Arrays.h"
+#include "../Distribution/Distribution.h"
+#include "../Distribution/DvmhDirective_func.h"
+
+#include "../Utils/errors.h"
+#include "../LoopAnalyzer/loop_analyzer.h"
+#include "directive_parser.h"
+#include "directive_creator.h"
+#include "../Utils/SgUtils.h"
+#include "../Sapfor.h"
+#include "../GraphLoop/graph_loops_func.h"
+#include "../Transformations/loop_transform.h"
+#include "../ExpressionTransform/expr_transform.h"
+#include "../GraphCall/graph_calls_func.h"
+
+#include "../Utils/AstWrapper.h"
+
+#define PRINT_DIR_RESULT 0
+
+#define FIRST(x)  get<0>(x)
+#define SECOND(x) get<1>(x)
+#define THIRD(x)  get<2>(x)
+
+using std::vector;
+using std::pair;
+using std::tuple;
+using std::map;
+using std::set;
+using std::make_pair;
+using std::make_tuple;
+using std::get;
+using std::string;
+using std::wstring;
+
+extern int sharedMemoryParallelization;
+
+static vector<pair<string, vector<Expression*>>> 
+    groupRealignsDirs(const vector<pair<string, vector<Expression*>>>& toRealign)
+{
+    map<pair<string, string>, vector<vector<Expression*>>> groupedRules;
+    for (auto& rule : toRealign)
+    {
+        auto currRule = rule.second;
+
+        string tRule = string(currRule[2]->unparse());
+        string arrRule = string(currRule[1]->unparse());
+        groupedRules[make_pair(tRule, arrRule)].push_back(currRule);
+    }
+
+    map<pair<string, string>, vector<Expression*>> mergedGroupedRules;
+    for (auto& rule : groupedRules)
+    {
+        SgExprListExp* mergedList = new SgExprListExp();
+        for (int z = 0; z < rule.second.size(); ++z)
+        {
+            if (z == 0)
+                mergedList->setLhs(rule.second[z][0]->GetOriginal());
+            else
+                mergedList->append(*rule.second[z][0]->GetOriginal());
+        }
+        vector<Expression*> medged = rule.second[0];
+        medged[0] = new Expression(mergedList);
+        mergedGroupedRules[rule.first] = medged;
+    }
+
+    vector<pair<string, vector<Expression*>>> retVal;
+    for (auto& elem : mergedGroupedRules)
+        retVal.push_back(make_pair("", elem.second));
+
+    return retVal;
+}
+
+//create realigns instead of full template redistribution
+pair<vector<Directive*>, vector<Directive*>>
+    createRealignRules(Statement* st, const uint64_t regId, File *file, const string &templClone, 
+                       const map<DIST::Array*, set<DIST::Array*>> &arrayLinksByFuncCalls, const set<DIST::Array*>& usedArrays,
+                       const pair<int, int> linesBeforeAfter)
+{
+    vector<vector<pair<string, vector<Expression*>>>> optimizedRules(2);
+    
+    for (int num = 0; num < 2; ++num)
+    {
+        for (auto &elemPair : sortArraysByName(usedArrays))
+        {
+            DIST::Array* elem = elemPair.second;
+            if (elem->IsNotDistribute())
+                continue;
+
+            auto realRef = getRealArrayRef(elem, regId, arrayLinksByFuncCalls);
+            auto rules = realRef->GetAlignRulesWithTemplate(regId);
+            auto links = realRef->GetLinksWithTemplate(regId);
+            const auto &templ = realRef->GetTemplateArray(regId);
+            if (templ == NULL)
+                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+            
+            vector<Expression*> realign = { NULL, NULL, NULL, NULL, NULL };
+            SgVarRefExp *ref = new SgVarRefExp((SgSymbol*)elem->GetNameInLocationS(st));
+
+            realign[0] = new Expression(ref);
+            SgExprListExp *list = new SgExprListExp();
+            string base = "iEX";
+            for (int z = 0; z < elem->GetDimSize(); ++z)
+            {
+                if (z == 0)
+                    list->setLhs(*new SgVarRefExp(findSymbolOrCreate(file, base + std::to_string(z))));
+                else
+                    list->append(*new SgVarRefExp(findSymbolOrCreate(file, base + std::to_string(z))));
+            }
+
+            realign[1] = new Expression(list);
+            if (num == 0)
+                realign[2] = new Expression(new SgArrayRefExp(*findSymbolOrCreate(file, templClone, new SgArrayType(*SgTypeInt()))));
+            else
+                realign[2] = new Expression(new SgArrayRefExp(*findSymbolOrCreate(file, templ->GetShortName(), new SgArrayType(*SgTypeInt()))));
+
+            vector<SgExpression*> templateRuleEx(templ->GetDimSize());
+            std::fill(templateRuleEx.begin(), templateRuleEx.end(), (SgExpression*)NULL);
+            for (int z = 0; z < elem->GetDimSize(); ++z)
+            {
+                if (links[z] != -1)
+                {
+                    SgExpression *toSet = NULL;
+                    auto symb = new SgVarRefExp(*findSymbolOrCreate(file, base + std::to_string(z)));
+                    if (rules[z] == make_pair(1, 0))
+                        toSet = symb;
+                    else if (rules[z].second == 0)
+                        toSet = &(*new SgValueExp(rules[z].first) * *symb);
+                    else if (rules[z].first == 1)
+                        toSet = &(*symb + *new SgValueExp(rules[z].second));
+                    else
+                        toSet = &(*new SgValueExp(rules[z].first) * *symb + *new SgValueExp(rules[z].second));
+                    templateRuleEx[links[z]] = toSet;
+                }
+            }
+
+            for (int z = 0; z < templateRuleEx.size(); ++z)
+            {
+                SgExpression *toSet = NULL;
+                if (templateRuleEx[z] == NULL)
+                    toSet = new SgVarRefExp(*findSymbolOrCreate(file, "*"));
+                else
+                    toSet = templateRuleEx[z];
+                ((SgArrayRefExp*)realign[2]->GetOriginal())->addSubscript(*toSet);
+
+            }
+
+            optimizedRules[num].push_back(make_pair("", realign));
+        }
+    }
+
+    vector<vector<pair<string, vector<Expression*>>>> groupedOptRules(2);
+    groupedOptRules[0] = groupRealignsDirs(optimizedRules[0]);
+    groupedOptRules[1] = groupRealignsDirs(optimizedRules[1]);
+
+    pair<vector<Directive*>, vector<Directive*>> retVal;
+    for (auto& elem : groupedOptRules[0])
+        retVal.first.push_back(new CreatedDirective(elem.first, elem.second, linesBeforeAfter.first));
+    for (auto& elem : groupedOptRules[1])
+        retVal.second.push_back(new CreatedDirective(elem.first, elem.second, linesBeforeAfter.second));
+    return retVal;
+}
+
+static bool hasFunctionCall(SgExpression* ex)
+{
+    bool ret = false;
+    if (ex)
+    {
+        if (ex->variant() == FUNC_CALL)
+            return true;
+        ret |= hasFunctionCall(ex->lhs());
+        ret |= hasFunctionCall(ex->rhs());
+    }
+    return ret;
+}
+
+static bool splitToBase(SgExpression *ex, pair<SgExpression*, int> &splited)
+{
+    bool res = true;
+    if (hasFunctionCall(ex))
+        return false;
+
+    if (ex->variant() == VAR_REF || ex->variant() == ARRAY_REF || ex->variant() == MULT_OP)
+        splited = make_pair(ex, 0);
+    else
+    {
+        if (ex->variant() == SUBT_OP || ex->variant() == ADD_OP)
+        {
+            int minus = (ex->variant() == ADD_OP) ? 1 : -1;
+            if (ex->rhs())
+            {
+                int err, val;
+                err = CalculateInteger(ex->rhs(), val);
+                if (err == 0)
+                {
+                    const int nextEx = ex->lhs()->variant();
+                    if (nextEx == VAR_REF || nextEx == ARRAY_REF || nextEx == MULT_OP)
+                        splited = make_pair(ex->lhs(), minus * val);
+                    else if (nextEx == SUBT_OP || nextEx == ADD_OP)
+                    {
+                        pair<SgExpression*, int> splitedNext;
+                        bool res = splitToBase(ex->lhs(), splitedNext);
+                        if (res == false)
+                            return false;
+                        else
+                            splited = make_pair(splitedNext.first, minus * val + splitedNext.second);
+                    }
+                    else
+                        return false;
+                }
+                else
+                    return false;
+            }
+            else
+                return false;
+        }
+        else
+            return false;
+    }
+    return res;
+}
+
+
+static void analyzeRightPart(SgExpression *ex, map<DIST::Array*, vector<pair<bool, map<string, pair<int, int>>>>> &rightValues,
+                             const map<DIST::Array*, vector<bool>> &dimsNotMatch)
+{
+    if (ex)
+    {
+        if (ex->variant() == ARRAY_REF)
+        {
+            const std::string name = ex->symbol()->identifier();
+            for (auto &elem : dimsNotMatch)
+            {
+                if (elem.first->GetShortName() == name)
+                {
+                    int idx = 0;
+                    for (auto expr = ex->lhs(); expr; expr = expr->rhs(), ++idx)
+                    {
+                        if (elem.second[idx])
+                        {
+                            int err, val;
+                            err = CalculateInteger(expr->lhs(), val);
+                            if (err == 0)
+                            {
+                                if (rightValues[elem.first][idx].first)
+                                {
+                                    auto it = rightValues[elem.first][idx].second.find("");
+                                    if (it == rightValues[elem.first][idx].second.end())
+                                        rightValues[elem.first][idx].second[""] = make_pair(val, val);
+                                    else
+                                    {
+                                        it->second.first = std::min(it->second.first, val);
+                                        it->second.second = std::max(it->second.second, val);
+                                    }
+                                }
+                                else
+                                {
+                                    rightValues[elem.first][idx].first = true;
+                                    rightValues[elem.first][idx].second[""] = make_pair(val, val);
+                                }
+                            }
+                            else
+                            {
+                                pair<SgExpression*, int> splited;
+                                bool result = splitToBase(expr->lhs(), splited);
+                                if (result)
+                                {
+                                    if (rightValues[elem.first][idx].first)
+                                    {
+                                        auto key = string(splited.first->unparse());
+                                        auto itS = rightValues[elem.first][idx].second.find(key);
+                                        if (itS == rightValues[elem.first][idx].second.end())
+                                            itS = rightValues[elem.first][idx].second.insert(itS, make_pair(key, make_pair(splited.second, splited.second)));
+                                        else
+                                        {
+                                            itS->second.first = std::min(itS->second.first, splited.second);
+                                            itS->second.second = std::max(itS->second.second, splited.second);
+                                        }
+                                    }
+                                    else
+                                    {
+                                        rightValues[elem.first][idx].first = true;
+                                        rightValues[elem.first][idx].second[string(splited.first->unparse())] = make_pair(splited.second, splited.second);                                     
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    break;
+                }
+            }
+        }
+
+        analyzeRightPart(ex->lhs(), rightValues, dimsNotMatch);
+        analyzeRightPart(ex->rhs(), rightValues, dimsNotMatch);
+    }
+}
+
+static bool analyzeLeftPart(SgExpression *left, const map<DIST::Array*, vector<bool>>& dimsNotMatch, 
+                            map<DIST::Array*, vector<pair<bool, pair<string, int>>>> &leftValues, string &base)
+{
+    const std::string name = left->symbol()->identifier();
+    for (auto& elem : dimsNotMatch)
+    {
+        if (elem.first->GetShortName() == name)
+        {
+            int idx = 0;
+            for (auto ex = left->lhs(); ex; ex = ex->rhs(), ++idx)
+            {
+                if (elem.second[idx])
+                {
+                    int err, val;
+                    err = CalculateInteger(ex->lhs(), val);
+                    if (err == 0)
+                    {
+                        if (leftValues[elem.first][idx].first)
+                        {
+                            if (leftValues[elem.first][idx].second.first != "") // has non zero base expression
+                                return false;
+                            if (leftValues[elem.first][idx].second.second != val) // has conflict writes
+                                return false;
+                        }
+                        else
+                            leftValues[elem.first][idx] = make_pair(true, make_pair("", val));
+                    }
+                    else // WRITE OP can not recognized
+                    {
+                        pair<SgExpression*, int> splited;
+                        bool result = splitToBase(ex->lhs(), splited);
+                        if (result == false)
+                            return false;
+                        if (leftValues[elem.first][idx].first)
+                        {
+                            // has conflict writes
+                            if (leftValues[elem.first][idx].second.first != string(splited.first->unparse()) ||
+                                leftValues[elem.first][idx].second.second != splited.second)
+                                return false;
+                        }
+                        else
+                        {
+                            base = string(splited.first->unparse());
+                            leftValues[elem.first][idx] = make_pair(true, make_pair(base, splited.second));
+                        }
+                    }
+                }
+            }
+            break;
+        }
+    }
+    return true;
+}
+
+bool analyzeLoopBody(LoopGraph* loopV, 
+                     map<DIST::Array*, vector<pair<bool, pair<string, int>>>>& leftValues,
+                     map<DIST::Array*, vector<pair<bool, map<string, pair<int, int>>>>>& rightValues,
+                     string& base,
+                     const map<DIST::Array*, vector<bool>> &dimsNotMatch,
+                     const map<string, FuncInfo*>& mapFuncInfo)
+{
+    SgStatement* loop = loopV->loop->GetOriginal();
+    for (auto st = loop; st != loop->lastNodeOfStmt(); st = st->lexNext())
+    {
+        if (st->variant() == ASSIGN_STAT)
+        {
+            auto left = st->expr(0);
+            if (left->variant() == ARRAY_REF)
+            {
+                bool ok = analyzeLeftPart(left, dimsNotMatch, leftValues, base);
+                if (ok == false)
+                    return false;
+            }
+            analyzeRightPart(st->expr(1), rightValues, dimsNotMatch);
+        }
+        else if (st->variant() == PROC_STAT)
+        {
+            string name = st->symbol()->identifier();
+            if (isIntrinsicFunctionName(name.c_str()) == 0)
+            {
+                //TODO: contains and modules
+                auto it = mapFuncInfo.find(name);
+                int z = 0;
+                for (SgExpression* ex = st->expr(0); ex; ex = ex->rhs(), ++z)
+                {
+                    if (ex->lhs()->variant() == ARRAY_REF)
+                    {
+                        bool ok = true;
+                        if (it == mapFuncInfo.end())
+                            ok = analyzeLeftPart(ex->lhs(), dimsNotMatch, leftValues, base);
+                        else
+                        {
+                            if (it->second->funcParams.isArgIn(z) && !it->second->funcParams.isArgOut(z))
+                                analyzeRightPart(ex->lhs(), rightValues, dimsNotMatch);
+                            else
+                                ok = analyzeLeftPart(ex->lhs(), dimsNotMatch, leftValues, base);
+                        }
+                        if (ok == false)
+                            return false;
+                    }
+                    else
+                        analyzeRightPart(ex->lhs(), rightValues, dimsNotMatch);
+                }
+            }
+        }
+        else
+        {
+            for (int i = 0; i < 3; ++i)
+                analyzeRightPart(st->expr(1), rightValues, dimsNotMatch);
+        }
+    }
+
+    //is OK ?
+    return true;
+}
+
+void createParallelDirs(File *file,
+                        map<string, vector<Directive*>>& createdDirectives,
+                        vector<Messages>& messages,
+                        const vector<LoopGraph*>& loopsInFile,
+                        const map<string, vector<FuncInfo*>>& allFuncInfo,
+                        const vector<ParallelRegion*>& parallelRegions,
+                        const map<LoopGraph*, void*>& depInfoForLoopGraph,
+                        const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls)
+{
+    const string file_name = file->filename();
+    map<int, LoopGraph*> mapLoopsInFile;
+    createMapLoopGraph(loopsInFile, mapLoopsInFile);
+
+    map<string, FuncInfo*> mapFuncInfo;
+    createMapOfFunc(allFuncInfo, mapFuncInfo);
+
+    for (int z = 0; z < parallelRegions.size(); ++z)
+    {
+        vector<Directive*> toInsert;
+
+        const DataDirective& dataDirectives = parallelRegions[z]->GetDataDir();
+        const vector<int>& currentVariant = parallelRegions[z]->GetCurrentVariant();
+        DIST::GraphCSR<int, double, attrType>& reducedG = parallelRegions[z]->GetReducedGraphToModify();
+        DIST::Arrays<int>& allArrays = parallelRegions[z]->GetAllArraysToModify();
+
+        auto& tmp = dataDirectives.distrRules;
+        vector<pair<DIST::Array*, const DistrVariant*>> currentVar;
+
+        if (sharedMemoryParallelization == 0)
+        {
+            for (int z1 = 0; z1 < currentVariant.size(); ++z1)
+                currentVar.push_back(make_pair(tmp[z1].first, &tmp[z1].second[currentVariant[z1]]));
+        }
+        else
+        {
+            for (auto& loop : mapLoopsInFile)
+            {
+                auto& rules = loop.second->getDataDir().distrRules;
+                for (auto& rule : rules)
+                    currentVar.push_back(make_pair(rule.first, &rule.second[0]));
+            }
+        }
+
+        selectParallelDirectiveForVariant(file, parallelRegions[z], reducedG, allArrays, loopsInFile, 
+                                          mapLoopsInFile, mapFuncInfo, currentVar,
+                                          toInsert, parallelRegions[z]->GetId(), arrayLinksByFuncCalls,
+                                          depInfoForLoopGraph, messages);
+
+        if (toInsert.size() > 0)
+        {
+            auto it = createdDirectives.find(file_name);
+            if (it == createdDirectives.end())
+                createdDirectives.insert(it, make_pair(file_name, toInsert));
+            else
+                for (int m = 0; m < toInsert.size(); ++m)
+                    it->second.push_back(toInsert[m]);
+        }
+    }
+}
+
+#undef PRINT_DIR_RESULT 
+#undef FIRST
+#undef SECOND
+#undef THIRD

src/DirectiveProcessing/directive_creator.h

@@ -0,0 +1,63 @@
+#pragma once
+
+#include "../Distribution/Distribution.h"
+#include "../Utils/errors.h"
+#include "../GraphLoop/graph_loops.h"
+#include "../Utils/types.h"
+
+void createParallelDirectives(const std::map<LoopGraph*, std::map<DIST::Array*, ArrayInfo*>> &loopInfo,
+                              const std::vector<ParallelRegion*>& regions, 
+                              const std::map<DIST::Array*, std::set<DIST::Array*>> &arrayLinksByFuncCalls,
+                              std::vector<Messages> &messagesForFile);
+
+void selectParallelDirectiveForVariant(File* file,
+                                       ParallelRegion* currReg,
+                                       DIST::GraphCSR<int, double, attrType>& reducedG,
+                                       DIST::Arrays<int>& allArrays,
+                                       const std::vector<LoopGraph*>& loopGraph,
+                                       const std::map<int, LoopGraph*>& mapLoopGraph,
+                                       const std::map<std::string, FuncInfo*>& mapFuncInfo,
+                                       const std::vector<std::pair<DIST::Array*, const DistrVariant*>>& distribution,
+                                       std::vector<Directive*>& toInsert,
+                                       const uint64_t regionId,
+                                       const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls,
+                                       const std::map<LoopGraph*, void*>& depInfoForLoopGraph,
+                                       std::vector<Messages>& messages);
+
+void filterParallelDirectives(const std::map<std::string, std::vector<LoopGraph*>>& loopGraph,
+                              std::map<std::string, std::vector<Directive*>>& createdDirectives);
+
+bool analyzeLoopBody(LoopGraph* loopV,
+                     std::map<DIST::Array*, std::vector<std::pair<bool, std::pair<std::string, int>>>>& leftValues,
+                     std::map<DIST::Array*, std::vector<std::pair<bool, std::map<std::string, std::pair<int, int>>>>>& rightValues,
+                     std::string& base,
+                     const std::map<DIST::Array*, std::vector<bool>>& dimsNotMatch,
+                     const std::map<std::string, FuncInfo*>& mapFuncInfo);
+
+std::pair<std::vector<Directive*>, std::vector<Directive*>> createRealignRules(Statement* spStat, const uint64_t regId, File* file, const std::string& templClone,
+                                                                               const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls, 
+                                                                               const std::set<DIST::Array*>& usedArrays,
+                                                                               const std::pair<int, int>);
+
+DIST::Array* getRealArrayRef(DIST::Array* in, const uint64_t regId, const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls);
+
+void shiftAlignRulesForTemplates(const std::set<DIST::Array*>& arrays, const uint64_t regId, DataDirective& dataDirectives, const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls);
+
+void createShadowSpec(const std::map<std::string, std::vector<LoopGraph*>>& loopGraph, const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls, const std::set<DIST::Array*>& forArrays);
+
+void addShadowFromAnalysis(ParallelDirective* dir, const std::map<DIST::Array*, ArrayInfo*>& currAccesses);
+
+bool checkForConflict(const std::map<DIST::Array*, ArrayInfo*>& currAccesses,
+                      const LoopGraph* currentLoop,
+                      std::map<DIST::Array*, std::pair<int, std::pair<int, int>>, DIST::ArrayComparator>& arrayWriteAcc,
+                      const std::vector<std::pair<std::pair<std::string, std::string>, std::vector<std::pair<int, int>>>>& acrossInfo,
+                      std::set<DIST::Array*>& acrossOutArrays);
+
+void createParallelDirs(File* file,
+                        std::map<std::string, std::vector<Directive*>>& createdDirectives,
+                        std::vector<Messages>& messages,
+                        const std::vector<LoopGraph*>& loopsInFile,
+                        const std::map<std::string, std::vector<FuncInfo*>>& allFuncInfo,
+                        const std::vector<ParallelRegion*>& parallelRegions,
+                        const std::map<LoopGraph*, void*>& depInfoForLoopGraph,
+                        const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls);

src/DirectiveProcessing/directive_omp_parser.cpp

@@ -0,0 +1,536 @@
+#include "../Utils/leak_detector.h"
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <cstdint>
+
+#include "dvm.h"
+#include "directive_omp_parser.h"
+#include "directive_parser.h"
+
+#include "../Utils/SgUtils.h"
+
+using std::vector;
+using std::map;
+using std::set;
+using std::string;
+
+
+void removeOmpDir(SgStatement* st)
+{
+    char* lineS = st->comments();
+    if (!lineS)
+        return;
+
+    vector<string> split;
+    splitString(lineS, '\n', split);
+
+    int idx = 0;
+    for (auto& elem : split)
+    {
+        string line = elem;
+        convertToLower(line);
+        if (line.substr(0, 5) == "!$omp")
+            lineS[idx + 1] = '_';
+        else if (line.substr(0, 3) == "!$ ")
+            lineS[idx + 1] = '_';
+        idx += line.size() + 1; // with '\n'
+    }
+}
+
+static inline void addToAttribute(SgStatement* st, int var, vector<SgExpression*> list)
+{
+    if (list.size())
+    {
+        SgExprListExp* ex = new SgExprListExp();
+        ex->setLhs(new SgExpression(var, makeExprList(list), NULL));
+        SgStatement* toAdd = new SgStatement(SPF_ANALYSIS_DIR, NULL, NULL, ex, NULL, NULL);
+        toAdd->setlineNumber(st->lineNumber());
+        toAdd->setLocalLineNumber(SPF_OMP_DIR);
+
+        //filter 
+        if (var == ACC_PRIVATE_OP)
+        {
+            vector<SgExpression*> list_new;
+
+            auto attributes = getAttributes<SgStatement*, SgStatement*>(st, set<int>{SPF_ANALYSIS_DIR});
+            set<string> privates;
+            for (auto& attr : attributes)
+                fillPrivatesFromComment(new Statement(attr), privates);
+
+            if (privates.size())
+            {
+                for (auto& elem : list)
+                    if (privates.find(elem->unparse()) == privates.end())
+                        list_new.push_back(elem);
+                list = list_new;
+
+                if (!list.size())
+                {
+                    __spf_print(1, "-- skip privates on line %d from OMP dir\n%s", st->lineNumber(), toAdd->unparse());
+                    return;
+                }
+            }
+        }
+        else if (var == REDUCTION_OP)
+        {
+            auto attributes = getAttributes<SgStatement*, SgStatement*>(st, set<int>{SPF_ANALYSIS_DIR});
+            map<string, set<string>> reduction;
+            for (auto& attr : attributes)
+                fillReductionsFromComment(new Statement(attr), reduction);
+
+            map<string, set<string>> reductionToAdd;
+            fillReductionsFromComment(new Statement(toAdd), reductionToAdd);
+
+            vector<SgExpression*> list_new;
+            if (reduction.size())
+            {
+                if (reduction == reductionToAdd)
+                {
+                    __spf_print(1, "-- skip reduction on line %d from OMP dir\n%s", st->lineNumber(), toAdd->unparse());
+                    return;
+                }
+
+                map<string, set<string>> reductionToAddNew;
+                for (auto& redPair : reductionToAdd)
+                {
+                    auto it = reduction.find(redPair.first);
+                    if (it == reduction.end())
+                        reductionToAddNew[redPair.first] = redPair.second;
+                    else
+                    {
+                        set<string> newVar;
+                        for (auto& var : redPair.second)
+                        {
+                            auto itVar = it->second.find(var);
+                            if (itVar == it->second.end())
+                                reductionToAddNew[redPair.first].insert(var);
+                        }
+                    }
+                }
+
+                if (!reductionToAddNew.size())
+                {
+                    __spf_print(1, "-- skip reduction on line %d from OMP dir\n%s", st->lineNumber(), toAdd->unparse());
+                    return;
+                }
+
+                if (reductionToAddNew != reductionToAdd)
+                {
+                    list.clear();
+                    for (auto& redPair : reductionToAddNew)
+                        for (auto& var : redPair.second)
+                            list.push_back(new SgExpression(ARRAY_OP,
+                                new SgKeywordValExp(redPair.first.c_str()),
+                                new SgVarRefExp(findSymbolOrCreate(current_file, var, NULL, getFuncStat(st)))));
+                }
+            }
+        }
+
+        ex = new SgExprListExp();
+        ex->setLhs(new SgExpression(var, makeExprList(list), NULL));
+        toAdd = new SgStatement(SPF_ANALYSIS_DIR, NULL, NULL, ex, NULL, NULL);
+
+        st->addAttribute(SPF_ANALYSIS_DIR, toAdd, sizeof(SgStatement));
+
+        if (var == ACC_PRIVATE_OP)
+            __spf_print(1, "-- set private attribute to line %d from OMP dir\n%s", st->lineNumber(), toAdd->unparse());
+        else if (var == REDUCTION_OP)
+            __spf_print(1, "-- set reduction attribute to line %d from OMP dir\n%s", st->lineNumber(), toAdd->unparse());
+    }
+}
+
+static bool is_write_in_do(SgStatement* st, const string& var)
+{
+    checkNull(st, convertFileName(__FILE__).c_str(), __LINE__);
+
+    if (st->variant() != FOR_NODE)
+        return false;
+
+    SgStatement* lastNode = st->lastNodeOfStmt();
+    for (SgStatement* op = st->lexNext(); st != lastNode; st = st->lexNext())
+    {
+        if (st->variant() == ASSIGN_STAT)
+        {
+            SgExpression* ex = st->expr(0);
+            if (ex->variant() == ARRAY_REF || ex->variant() == VAR_REF)
+                if (var == ex->symbol()->identifier())
+                    return true;
+        }
+        else if (st->variant() == FOR_NODE)
+        {
+            if (var == isSgForStmt(st)->doName()->identifier())
+                return true;
+        }
+    }
+    return false;
+}
+
+vector<OmpDir> parseOmpInStatement(SgStatement* st, const set<string>& globalPriv, bool forDo)
+{
+    vector<OmpDir> resultAll;
+
+    const char* lineS = st->comments();
+    if (!lineS)
+        return resultAll;
+
+    string comment(lineS);
+    convertToLower(comment);
+
+    vector<string> split;
+    splitString(comment, '\n', split);
+
+    for (int z = split.size() - 1; z >= 0; z--)
+    {
+        string line = split[z];
+
+        if (line.substr(0, 6) == "!$omp&")
+        {
+            if (z - 1 < 0)
+                break;
+            split[z - 1] += line.substr(6);
+            split[z] = "";
+        }
+    }
+
+    for (auto& line : split)
+    {
+        if (line.substr(0, 5) == "!$omp")
+        {
+            OmpDir result;
+
+            string line1 = "";
+            int space = 0;
+            int brake = 0;
+            for (int z = 0; z < line.size(); ++z)
+            {
+                if (brake < 0)
+                    return vector<OmpDir>(); // error
+
+                if (brake == 0)
+                {
+                    if (line[z] == ' ')
+                        space++;
+                    else
+                        space = 0;
+                    if ((line[z] == ' ' && space <= 1) || line[z] != ' ')
+                        line1 += line[z];
+                }
+                else
+                {
+                    if (line[z] != ' ')
+                        line1 += line[z];
+                }
+
+                if (line[z] == '(')
+                {
+                    while (line1.size() > 2 && line1[line1.size() - 2] == ' ')
+                        line1 = line1.erase(line1.size() - 2, 1);
+                    brake++;
+                    space = 0;
+                }
+                else if (line[z] == ')')
+                    brake--;
+            }
+            vector<string> lexems;
+            splitString(line1, ' ', lexems);
+            bool doLexem = false;
+            bool end = false;
+            bool parallel = false;
+            bool privat = false;
+
+            for (auto& lexem : lexems)
+            {
+                if (lexem == "do")
+                {
+                    doLexem = true;
+                    result.keys.insert(lexem);
+                }
+                if (lexem == "end")
+                {
+                    end = true;
+                    result.keys.insert(lexem);
+                }
+                if (lexem == "parallel")
+                {
+                    parallel = true;
+                    result.keys.insert(lexem);
+                }
+                if (lexem == "private")
+                {
+                    privat = true;
+                    result.keys.insert(lexem);
+                }
+            }
+
+            if (privat == false)
+            {
+                if (forDo && doLexem)
+                {
+                    vector<SgExpression*> list;
+                    for (auto& var : globalPriv)
+                        if (is_write_in_do(st, var))
+                            list.push_back(new SgVarRefExp(findSymbolOrCreate(current_file, var, NULL, getFuncStat(st))));
+
+                    if (list.size())
+                        addToAttribute(st, ACC_PRIVATE_OP, list);
+                }
+            }
+
+            for (auto& lexem : lexems)
+            {
+                bool priv = lexem.substr(0, strlen("private(")) == "private(";
+                bool threadpriv = lexem.substr(0, strlen("threadprivate(")) == "threadprivate(";
+                bool red = lexem.substr(0, strlen("reduction(")) == "reduction(";
+                if (priv || threadpriv)
+                {
+                    vector<string> sublex;
+                    splitString(lexem, '(', sublex);
+                    if (sublex.size() == 2 && lexem.back() == ')')
+                    {
+                        splitString(sublex[1].erase(sublex[1].size() - 1), ',', sublex);
+
+                        vector<SgExpression*> list;
+                        set<string> uniqList;
+                        for (auto& varG : globalPriv)
+                            uniqList.insert(varG);
+
+                        for (auto& var : sublex)
+                            uniqList.insert(var);
+
+                        for (auto& var : uniqList)
+                        {
+                            if (priv)
+                            {
+                                result.privVars.insert(var);
+                                list.push_back(new SgVarRefExp(findSymbolOrCreate(current_file, var, NULL, getFuncStat(st))));
+                            }
+                            else
+                                result.threadPrivVars.insert(var);
+                        }
+
+                        if (forDo && doLexem && priv)
+                            addToAttribute(st, ACC_PRIVATE_OP, list);
+                    }
+                }
+                else if (red)
+                {
+                    vector<string> sublex;
+                    splitString(lexem, '(', sublex);
+                    if (sublex.size() == 2 && lexem.back() == ')')
+                    {
+                        splitString(sublex[1].erase(sublex[1].size() - 1), ':', sublex);
+
+                        vector<string> vars;
+                        vector<SgExpression*> list;
+                        splitString(sublex[1], ',', vars);
+                        string op = "";
+
+                        if (sublex[0] == "+")
+                            op = "sum";
+                        else if (sublex[0] == "*")
+                            op = "prod";
+                        else if (sublex[0] == "max")
+                            op = "max";
+                        else if (sublex[0] == "min")
+                            op = "min";
+                        else if (sublex[0] == ".or." || sublex[0] == "or")
+                            op = "or";
+                        else if (sublex[0] == ".and." || sublex[0] == "and")
+                            op = "and";
+                        else if (sublex[0] == ".eqv." || sublex[0] == "eqv")
+                            op = "eqv";
+                        else if (sublex[0] == ".neqv." || sublex[0] == "neqv")
+                            op = "neqv";
+
+                        if (op != "")
+                        {
+                            for (auto& var : vars)
+                            {
+                                result.redVars[sublex[0]].insert(var);
+                                list.push_back(new SgExpression(ARRAY_OP, new SgKeywordValExp(op.c_str()), new SgVarRefExp(findSymbolOrCreate(current_file, var, NULL, getFuncStat(st)))));
+                            }
+                        }
+
+                        if (forDo && doLexem && op != "")
+                            addToAttribute(st, REDUCTION_OP, list);
+                    }
+                }
+            }
+
+            resultAll.push_back(result);
+        }
+    }
+
+    return resultAll;
+}
+
+//TODO: need to use IR and RD for checking
+static void filterPrivates(OmpDir& dir)
+{
+    if (dir.privVars.size() == 0)
+        return;
+
+    for (auto st = dir.start; st != dir.end; st = st->lexNext())
+    {
+        vector<OmpDir> res;
+        if (st != dir.start)
+        {
+            set<string> dummy;
+            res = parseOmpInStatement(st, dummy);
+        }
+
+        bool hasParallelDo = false;
+        for (auto& dir : res)
+        {
+            if (dir.keys.find("parallel") != dir.keys.end() ||
+                dir.keys.find("do") != dir.keys.end())
+            {
+                hasParallelDo = true;
+            }
+        }
+
+        if (res.size() == 0 || !hasParallelDo)
+        {
+            if (st->variant() == ASSIGN_STAT)
+            {
+                if (st->expr(0))
+                {
+                    string ref = st->expr(0)->symbol()->identifier();
+                    dir.privVars.erase(ref);
+                }
+            }
+        }
+    }
+}
+
+static vector<OmpDir> findAllGlobalParallelRegions(SgStatement* stFunc)
+{
+    vector<OmpDir> sections;
+
+    SgStatement* lastNode = stFunc->lastNodeOfStmt();
+    for (auto st = stFunc; st != lastNode; st = st->lexNext())
+    {
+        if (st == NULL)
+        {
+            __spf_print(1, "internal error in analysis, parallel directives will not be generated for this file!\n");
+            break;
+        }
+
+        if (st->variant() == CONTAINS_STMT)
+            break;
+
+        set<string> dummy;
+        auto res = parseOmpInStatement(st, dummy);
+
+        for (auto& dir : res)
+        {
+            auto end = dir.keys.end();
+            if (dir.keys.find("parallel") != end
+                && dir.keys.find("do") == end
+                && dir.keys.find("end") == end)
+            {
+                if (sections.size() && sections.back().end == NULL) // has open parallel region
+                {
+                    __spf_print(1, "wrong omp directives placed on line %d\n", st->lineNumber());
+                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+                }
+
+                sections.push_back(dir);
+                sections.back().start = st;
+            }
+            else if (dir.keys.find("parallel") != end
+                     && dir.keys.find("do") == end
+                     && dir.keys.find("end") != end)
+            {
+                if (!sections.size())
+                {
+                    __spf_print(1, "wrong omp directives placed on line %d\n", st->lineNumber());
+                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+                }
+                sections.back().end = st;
+            }
+        }
+    }
+
+    for (auto& dir : sections)
+        filterPrivates(dir);
+
+    return sections;
+}
+
+static set<string> getGlobalPrivate(SgStatement* st, const vector<OmpDir>& globalParallelRegions)
+{
+    set<string> globalPrivates;
+    const int line = st->lineNumber();
+    if (line > 0)
+    {
+        for (auto& reg : globalParallelRegions)
+        {
+            if (reg.start->lineNumber() <= line && line < reg.end->lineNumber())
+            {
+                if (reg.privVars.size())
+                    return reg.privVars;
+                else
+                    return globalPrivates;
+            }
+        }
+    }
+    else
+    {
+        for (auto& reg : globalParallelRegions)
+        {
+            for (auto stF = reg.start; stF != reg.end; stF = stF->lexNext())
+            {
+                if (st == stF)
+                {
+                    if (reg.privVars.size())
+                        return reg.privVars;
+                    else
+                        return globalPrivates;
+                }
+            }
+        }
+    }
+
+    return globalPrivates;
+}
+
+void parseOmpDirectives(SgFile* file, vector<Messages>& currMessages)
+{
+    int funcNum = file->numberOfFunctions();
+
+    for (int i = 0; i < funcNum; ++i)
+    {
+        SgStatement* st = file->functions(i);
+        SgStatement* lastNode = st->lastNodeOfStmt();
+
+        vector<OmpDir> globalParallelRegions = findAllGlobalParallelRegions(st);
+
+        while (st != lastNode)
+        {
+            if (st == NULL)
+            {
+                __spf_print(1, "internal error in analysis, parallel directives will not be generated for this file!\n");
+                break;
+            }
+
+            if (st->variant() == CONTAINS_STMT)
+                break;
+           
+            if (st->variant() == FOR_NODE)
+            {
+                SgForStmt* currSt = (SgForStmt*)st;
+                if (currSt->isEnddoLoop() == 0)
+                {
+                    __spf_print(1, "wrong omp directives placed\n");
+                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+                }
+                else
+                    parseOmpInStatement(st, getGlobalPrivate(st, globalParallelRegions), true);
+            }
+            st = st->lexNext();
+        }
+    }
+}

src/DirectiveProcessing/directive_omp_parser.h

@@ -0,0 +1,26 @@
+#pragma once
+#include <string>
+#include <set>
+#include <map>
+#include <vector>
+
+#include "../Utils/errors.h"
+
+#define SPF_USER_DIR 777
+#define SPF_USER_DIR_COPY 999
+#define SPF_OMP_DIR  888
+
+struct OmpDir
+{
+    std::set<std::string> privVars;
+    std::set<std::string> threadPrivVars;
+    std::map<std::string, std::set<std::string>> redVars;
+    std::set<std::string> keys;
+
+    SgStatement* start = NULL;
+    SgStatement* end = NULL;
+};
+
+void removeOmpDir(SgStatement* st);
+std::vector<OmpDir> parseOmpInStatement(SgStatement* st, const std::set<std::string>& globalPriv, bool forDo = false);
+void parseOmpDirectives(SgFile* file, std::vector<Messages>& currMessages);

src/DirectiveProcessing/directive_parser.cpp

@@ -0,0 +1,656 @@
+#include "../Utils/leak_detector.h"
+
+#include <cstdio>
+#include <cstring>
+#include <cstring>
+#include <cstdlib>
+#include <string>
+#include <vector>
+#include <locale>
+#include <set>
+#include <map>
+#include <assert.h>
+
+#include "directive_parser.h"
+#include "../Utils/SgUtils.h"
+#include "../LoopAnalyzer/loop_analyzer.h"
+#include "../Utils/AstWrapper.h"
+#include "../Utils/errors.h"
+
+using std::string;
+using std::vector;
+using std::set;
+using std::tuple;
+using std::map;
+using std::make_pair;
+using std::pair;
+
+bool isSPF_NoInline(Statement *stIn)
+{
+    if (stIn)
+    {
+        SgStatement *st = stIn->GetOriginal();
+        for (auto &data : getAttributes<SgStatement*, SgStatement*>(st, set<int>{ SPF_TRANSFORM_DIR }))
+        {
+            SgExpression *exprList = data->expr(0);
+            while (exprList)
+            {
+                if (exprList->lhs()->variant() == SPF_NOINLINE_OP)
+                {
+                    //__spf_print(1, "found no inline\n");
+                    return true;
+                }
+                exprList = exprList->rhs();
+            }
+        }
+    }
+    return false;
+}
+
+static map<SgSymbol*, Symbol*> dictCreated;
+
+static inline string getSymbol(const string& s) { return s; }
+static inline string getSymbol(SgSymbol* s) { return s->identifier(); }
+
+static inline string getData(SgExpression *symb, string*, bool moduleNameAdd = false) 
+{
+    SgSymbol* base = symb->symbol();
+    SgSymbol* symbOr = OriginalSymbol(symb->symbol());
+    if (symbOr == base)
+        return symbOr->identifier();
+    else
+    {
+        SgStatement* scope = symbOr->scope();
+        checkNull(scope, convertFileName(__FILE__).c_str(), __LINE__);
+        if (scope->variant() != MODULE_STMT)
+            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+        return string(scope->symbol()->identifier()) + "::" + symbOr->identifier();
+    }
+}
+
+static inline Expression* getData(SgExpression *symb, Expression**, bool moduleNameAdd = false) 
+{ 
+    return new Expression(symb);
+}
+
+static inline Symbol* getData(SgExpression *symb, Symbol**, bool moduleNameAdd = false)
+{
+    SgSymbol *symbOr = OriginalSymbol(symb->symbol());
+    auto it = dictCreated.find(symbOr);
+    if (it == dictCreated.end())
+        it = dictCreated.insert(it, make_pair(symbOr, new Symbol(symbOr)));
+    return it->second;
+}
+
+template<typename fillType>
+void fillPrivatesFromComment(Statement *stIn, set<fillType> &privates, int type)
+{
+    if (stIn)
+    {
+        SgStatement *st = stIn->GetOriginal();
+        if (st->variant() == SPF_ANALYSIS_DIR)
+        {
+            SgExpression *exprList = st->expr(0);
+            while (exprList)
+            {
+                const int var = exprList->lhs()->variant();
+                if ( ((var == ACC_PRIVATE_OP || var == SPF_PROCESS_PRIVATE_OP) && type == -1)  ||
+                     (var == ACC_PRIVATE_OP && var == type) ||
+                     (var == SPF_PROCESS_PRIVATE_OP && var == type) )
+                {                    
+                    SgExpression *list = exprList->lhs()->lhs();
+                    while (list)
+                    {
+                        fillType *dummy = NULL;
+                        privates.insert(getData(list->lhs(), dummy));
+                        list = list->rhs();
+                    }
+                    break;
+                }
+                exprList = exprList->rhs();
+            }
+        }
+    }
+}
+
+template void fillPrivatesFromComment(Statement *st, set<string> &privates, int type);
+template void fillPrivatesFromComment(Statement *st, set<Symbol*> &privates, int type);
+
+//XXX: need to remove message and to add implementation
+extern map<string, vector<Messages>> SPF_messages;
+
+//for simple reduction
+template<typename fillType>
+void fillReductionsFromComment(Statement *stIn, map<string, set<fillType>> &reduction, bool moduleNameAdd, int type)
+{    
+    bool error = false;
+    if (stIn)
+    {
+        SgStatement *st = stIn->GetOriginal();
+        if (st->variant() == type)
+        {
+            SgExpression* exprList = NULL;
+            if (type == SPF_ANALYSIS_DIR)
+                exprList = st->expr(0);
+            else if (type == DVM_PARALLEL_ON_DIR)
+                exprList = st->expr(1);
+
+            while (exprList)
+            {
+                if (exprList->lhs()->variant() == REDUCTION_OP)
+                {
+                    SgExpression *list = exprList->lhs()->lhs();                    
+                    while (list)
+                    {                        
+                        SgExpression *currRed = list->lhs(); // with variant ARRAY_OP
+                        
+                        fillType redSymb, *dummy = NULL;
+                        //minloc/maxloc
+                        if (currRed->rhs()->variant() == EXPR_LIST)
+                            redSymb = getData(currRed->rhs()->lhs(), dummy, moduleNameAdd);
+                        else
+                        {
+                            redSymb = getData(currRed->rhs(), dummy, moduleNameAdd);
+                            if (currRed->rhs()->variant() == ARRAY_REF)
+                            {
+                                SgExpression* ref = currRed->rhs();
+                                if (ref->lhs() != NULL || ref->rhs() != NULL)
+                                {
+                                    //XXX
+                                    std::wstring messageE, messageR;
+                                    __spf_printToLongBuf(messageE, L"Reduction by element of array '%s' is not supported", to_wstring(getSymbol(redSymb)).c_str());
+                                    __spf_printToLongBuf(messageR, R182, to_wstring(getSymbol(redSymb)).c_str());
+
+                                    SPF_messages[current_file->filename()].push_back(Messages(ERROR, stIn->lineNumber(), messageR, messageE, 1059));
+                                    error = true;
+                                }
+                            }
+                        }
+                        string oper = ((SgKeywordValExp *)(currRed->lhs()))->value();
+                        
+                        auto it = reduction.find(oper);
+                        if (oper == "minloc" || oper == "maxloc")
+                        {
+                            //skip
+                            //__spf_print(1, "  MAXLOC/MINLOC operation from SPF not supported yet, ignored\n");
+                        }
+                        else
+                        {
+                            if (it == reduction.end())
+                                it = reduction.insert(it, make_pair(oper, set<fillType>()));
+                            it->second.insert(redSymb);
+                        }
+                        list = list->rhs();
+                    }
+                }
+                exprList = exprList->rhs();
+            }
+        }
+    }
+
+    if (error)
+        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+}
+
+template void fillReductionsFromComment(Statement *st, map<string, set<string>> &reduction, bool, int);
+template void fillReductionsFromComment(Statement *st, map<string, set<Symbol*>> &reduction, bool, int);
+
+//for min/max loc reduction
+template<typename fillType>
+void fillReductionsFromComment(Statement *stIn, map<string, set<tuple<fillType, fillType, int>>> &reduction, bool moduleNameAdd, int type)
+{
+    if (stIn)
+    {
+        SgStatement *st = stIn->GetOriginal();
+        if (st->variant() == type)
+        {
+            SgExpression* exprList = NULL;
+            if (type == SPF_ANALYSIS_DIR)
+                exprList = st->expr(0);
+            else if (type == DVM_PARALLEL_ON_DIR)
+                exprList = st->expr(1);
+
+            while (exprList)
+            {
+                if (exprList->lhs()->variant() == REDUCTION_OP)
+                {
+                    SgExpression *list = exprList->lhs()->lhs();
+                    while (list)
+                    {
+                        SgExpression *currRed = list->lhs();
+
+                        fillType redSymb, *dummy = NULL;
+                        //minloc/maxloc
+                        if (currRed->rhs()->variant() == EXPR_LIST)
+                            redSymb = getData(currRed->rhs()->lhs(), dummy, moduleNameAdd);
+                        else
+                            redSymb = getData(currRed->rhs(), dummy, moduleNameAdd);
+                        string oper = ((SgKeywordValExp *)(currRed->lhs()))->value();
+
+                        auto it = reduction.find(oper);
+                        if (oper == "minloc" || oper == "maxloc")
+                        {
+                            fillType arraySymb = getData(currRed->rhs()->rhs()->lhs(), dummy);
+                            int dim = currRed->rhs()->rhs()->rhs()->lhs()->valueInteger();
+
+                            if (it == reduction.end())
+                                it = reduction.insert(it, make_pair(oper, set<tuple<fillType, fillType, int>>()));
+                            it->second.insert(std::make_tuple(redSymb, arraySymb, dim));
+                        }
+                        else
+                        {
+                            //skip
+                        }
+                        list = list->rhs();
+                    }
+                }
+                exprList = exprList->rhs();
+            }
+        }
+    }
+}
+
+template void fillReductionsFromComment(Statement *st, map<string, set<tuple<string, string, int>>> &reduction, bool, int);
+template void fillReductionsFromComment(Statement *st, map<string, set<tuple<Symbol*, Symbol*, int>>> &reduction, bool, int);
+
+void fillParameterFromComment(Statement *stIn, vector<pair<Expression*, Expression*>> &assigns)
+{
+    if (stIn)
+    {
+        SgStatement *st = stIn->GetOriginal();
+        if (st->variant() == SPF_ANALYSIS_DIR)
+        {
+            SgExpression *exprList = st->expr(0);
+            while (exprList)
+            {
+                if (exprList->lhs() && exprList->lhs()->variant() == SPF_PARAMETER_OP)
+                {
+                    auto paramList = exprList->lhs()->lhs();
+                    while (paramList)
+                    {
+                        assigns.push_back(make_pair(new Expression(paramList->lhs()->lhs()), new Expression(paramList->lhs()->rhs())));
+                        paramList = paramList->rhs();
+                    }
+                }
+                exprList = exprList->rhs();
+            }
+        }
+    }
+}
+
+template<typename fillType>
+static void fillShadowAcross(const int type, Statement *stIn, vector<pair<pair<fillType, string>, vector<pair<int, int>>>> &data, set<fillType> *corner = NULL)
+{
+    if (stIn)
+    {
+        SgStatement *st = stIn->GetOriginal();
+        for (int list = 0; list < 3; ++list)
+        {
+            SgExpression *exprList = st->expr(list);
+            while (exprList)
+            {
+                if (exprList->lhs()->variant() == type)
+                {
+                    SgExpression *listExp = exprList->lhs()->lhs();
+                    while (listExp)
+                    {
+                        SgExpression *list = listExp;
+                        bool needCornerAdd = false;
+                        if (list->lhs()->variant() == ARRAY_OP)
+                        {
+                            list = list->lhs();
+                            needCornerAdd = true;
+                        }
+
+                        SgSymbol *symb = OriginalSymbol(list->lhs()->symbol());
+
+                        fillType arrayName, *dummy = NULL;
+                        arrayName = getData(list->lhs(), dummy);
+                        bool cond = false;
+
+                        if (corner && needCornerAdd)
+                            corner->insert(arrayName);
+
+                        pair<pair<fillType, string>, vector<pair<int, int>>> *toAdd;
+                        for (int i = 0; i < data.size(); ++i)
+                        {
+                            if (data[i].first.first == arrayName)
+                            {
+                                toAdd = &data[i];
+                                cond = true;
+                                break;
+                            }
+                        }
+
+                        if (!cond)
+                        {
+                            auto uniqKey = getFromUniqTable(symb);
+                            data.push_back(make_pair(make_pair(arrayName, getShortName(uniqKey)), vector<pair<int, int>>()));
+                            toAdd = &data.back();
+                        }
+
+                        SgExpression *listEx = list->lhs()->lhs();
+                        int idx = 0;
+                        while (listEx)
+                        {
+                            SgExpression *width = listEx->lhs();
+                            const int left = width->lhs()->valueInteger();
+                            const int right = width->rhs()->valueInteger();
+                            if (toAdd->second.size() <= idx)
+                                toAdd->second.push_back(make_pair(left, right));
+                            else
+                            {
+                                toAdd->second[idx].first = std::max(toAdd->second[idx].first, left);
+                                toAdd->second[idx].second = std::max(toAdd->second[idx].second, right);
+                            }
+                            idx++;
+                            listEx = listEx->rhs();
+                        }
+
+                        listExp = listExp->rhs();
+                    }
+                }
+                exprList = exprList->rhs();
+            }
+        }
+    }
+}
+
+
+template<typename fillType>
+void fillShadowAcrossFromComment(const int type, Statement *stIn, vector<pair<pair<fillType, string>, vector<pair<int, int>>>> &data)
+{
+    if (stIn)
+        if (stIn->GetOriginal()->variant() == SPF_PARALLEL_DIR)
+            fillShadowAcross(type, stIn, data);
+}
+
+template void fillShadowAcrossFromComment(const int type, Statement *st, vector<pair<pair<string, string>, vector<pair<int, int>>>> &data);
+template void fillShadowAcrossFromComment(const int type, Statement *st, vector<pair<pair<Symbol*, string>, vector<pair<int, int>>>> &data);
+
+
+template<typename fillType>
+void fillShadowAcrossFromParallel(const int type, Statement *stIn, vector<pair<pair<fillType, string>, vector<pair<int, int>>>> &data, set<fillType> &corner)
+{
+    if (stIn)
+        if (stIn->GetOriginal()->variant() == DVM_PARALLEL_ON_DIR)
+            fillShadowAcross(type, stIn, data, &corner);
+}
+
+template void fillShadowAcrossFromParallel(const int type, Statement *st, vector<pair<pair<string, string>, vector<pair<int, int>>>> &data, set<string> &corner);
+template void fillShadowAcrossFromParallel(const int type, Statement* st, vector<pair<pair<Symbol*, string>, vector<pair<int, int>>>>& data, set<Symbol*> &corner);
+
+
+template<typename fillType>
+void fillRemoteFromComment(Statement *stIn, map<pair<fillType, string>, Expression*> &remote, bool isFull, int type)
+{
+    if (stIn)
+    {
+        SgStatement *st = stIn->GetOriginal();
+        if (st->variant() == type)
+        {
+            SgExpression *exprList = NULL;
+            if (type == SPF_PARALLEL_DIR || type == DVM_REMOTE_ACCESS_DIR)
+                exprList = st->expr(0);
+            else if (type == DVM_PARALLEL_ON_DIR)
+                exprList = st->expr(1);            
+
+            while (exprList)
+            {
+                if (exprList->lhs()->variant() == REMOTE_ACCESS_OP || type == DVM_REMOTE_ACCESS_DIR)
+                {
+                    SgExpression *list;
+                    if (type == DVM_REMOTE_ACCESS_DIR)
+                        list = exprList;
+                    else
+                        list = exprList->lhs()->lhs();
+
+                    while (list)
+                    {
+                        fillType arrayName, *dummy = NULL;
+                        arrayName = getData(list->lhs(), dummy);
+                        
+                        char *str;
+                        if (list->lhs()->lhs())
+                            str = list->lhs()->lhs()->unparse();
+                        else
+                            str = "";
+
+                        if (isFull)
+                            remote.insert(make_pair(make_pair(arrayName, string(str)), new Expression(list->lhs())));
+                        else
+                        {
+                            if (list->lhs()->lhs())
+                                remote.insert(make_pair(make_pair(arrayName, string(str)), new Expression(list->lhs()->lhs())));
+                            else
+                                remote.insert(make_pair(make_pair(arrayName, string(str)), new Expression(new SgExprListExp())));
+                        }
+                        list = list->rhs();
+                    }
+                }
+
+                if (type == DVM_REMOTE_ACCESS_DIR)
+                    break;
+                exprList = exprList->rhs();
+            }
+        }
+    }
+}
+
+template void fillRemoteFromComment(Statement *st, map<pair<string, string>, Expression*> &remote, bool isFull, int type);
+template void fillRemoteFromComment(Statement *st, map<pair<Symbol*, string>, Expression*> &remote, bool isFull, int type);
+template void fillRemoteFromComment(Statement *st, map<pair<Expression*, string>, Expression*> &remote, bool isFull, int type);
+
+void fillAcrossInfoFromDirectives(const LoopGraph *loopInfo, vector<pair<pair<string, string>, vector<pair<int, int>>>> &acrossInfo)
+{
+    SgForStmt *currentLoop = (SgForStmt*)loopInfo->loop;
+    for (auto &data : getAttributes<SgStatement*, SgStatement*>(currentLoop, set<int>{ SPF_ANALYSIS_DIR, SPF_PARALLEL_DIR, SPF_TRANSFORM_DIR }))
+        fillShadowAcrossFromComment(ACROSS_OP, new Statement(data), acrossInfo);
+}
+
+void fillFissionPrivatesExpansionFromComment(Statement *stIn, vector<string> &vars)
+{
+    if (stIn)
+    {
+        SgStatement *st = stIn->GetOriginal();
+        if (st->variant() == SPF_TRANSFORM_DIR)
+        {
+            SgExpression *exprList = st->expr(0);
+            while (exprList)
+            {
+                if (exprList->lhs() && (exprList->lhs()->variant() == SPF_FISSION_OP || exprList->lhs()->variant() == SPF_EXPAND_OP))
+                {
+                    SgExpression *list = exprList->lhs()->lhs();
+                    while (list)
+                    {
+                        if (list->lhs()->variant() == VAR_REF)
+                            vars.push_back(list->lhs()->symbol()->identifier());
+
+                        list = list->rhs();
+                    }
+                }
+                exprList = exprList->rhs();
+            }
+        }
+    }
+}
+
+template<typename fillType>
+void fillShrinkFromComment(Statement *stIn, vector<pair<fillType, vector<int>>> &varDims)
+{
+    if (stIn)
+    {
+        SgStatement *st = stIn->GetOriginal();
+        if (st->variant() == SPF_TRANSFORM_DIR)
+        {
+            SgExpression *exprList = st->expr(0);
+            while (exprList)
+            {
+                if (exprList->lhs() && (exprList->lhs()->variant() == SPF_SHRINK_OP))
+                {
+                    SgExpression *list = exprList->lhs()->lhs();
+                    while (list)
+                    {
+                        // get identifier
+                        fillType var, *dummy = NULL; 
+                        var = getData(list->lhs(), dummy);
+
+                        vector<int> dims;
+                        SgExpression *dimList = list->lhs()->lhs();
+                        while (dimList)
+                        {
+                            // filling dimensions
+                            dimList->lhs()->isInteger() ? dims.push_back(dimList->lhs()->valueInteger()) : dims.push_back(-1);
+                            dimList = dimList->rhs();
+                        }
+
+                        varDims.push_back(make_pair(var, dims));
+
+                        list = list->rhs();
+                    }
+                }
+
+                exprList = exprList->rhs();
+            }
+        }
+    }
+}
+
+template void fillShrinkFromComment(Statement *stIn, vector<pair<Symbol*, vector<int>>> &varDims);
+template void fillShrinkFromComment(Statement *stIn, vector<pair<string, vector<int>>> &varDims);
+
+template<typename fillType>
+void fillCheckpointFromComment(Statement *stIn, map<int, Expression*> &clauses, set<fillType> &vars, set<fillType> &expt)
+{
+    if (stIn)
+    {
+        SgStatement *st = stIn->GetOriginal();
+        if (st->variant() == SPF_CHECKPOINT_DIR)
+        {
+            SgExpression *exprList= st->expr(0);
+            while (exprList)
+            {
+                if (exprList->lhs())
+                {
+                    SgExpression *toInsert = NULL;
+                    if (exprList->lhs()->variant() == SPF_INTERVAL_OP)
+                        toInsert = new SgExpression(EXPR_LIST, exprList->lhs(), NULL);
+                    else if (exprList->lhs()->variant() == SPF_FILES_COUNT_OP)
+                        toInsert = new SgExpression(EXPR_LIST, exprList->lhs()->lhs(), NULL);
+                    else
+                        toInsert = exprList->lhs()->lhs();
+                    auto it = clauses.find(exprList->lhs()->variant());
+                    if (it == clauses.end())
+                        it = clauses.insert(it, make_pair(exprList->lhs()->variant(), new Expression(toInsert)));
+                    else
+                    {
+                        auto expr = it->second->GetOriginal();
+                        while (expr && expr->rhs())
+                            expr = expr->rhs();
+                        expr->setRhs(toInsert);
+                    }
+                    if (exprList->lhs()->variant() == SPF_VARLIST_OP ||
+                        exprList->lhs()->variant() == SPF_EXCEPT_OP)
+                    {
+                        auto expr = exprList->lhs()->lhs();
+                        while (expr)
+                        {
+                            // get identifier
+                            fillType var, *dummy = NULL; 
+                            var = getData(expr->lhs(), dummy);
+                            if (exprList->lhs()->variant() == SPF_VARLIST_OP)
+                            {
+                                auto it = vars.find(var);
+                                if (it == vars.end())
+                                    vars.insert(var);
+                            }
+                            else
+                            {
+                                auto it = expt.find(var);
+                                if (it == expt.end())
+                                    expt.insert(var);
+                            }
+                            expr = expr->rhs();
+                        }
+                    }
+                }
+                exprList = exprList->rhs();
+            }
+        }
+    }
+}
+
+template void fillCheckpointFromComment(Statement *stIn, map<int, Expression*> &clauses, set<Symbol*> &vars, set<Symbol*> &expt);
+template void fillCheckpointFromComment(Statement *stIn, map<int, Expression*> &clauses, set<string> &vars, set<string> &expt);
+
+void fillInfoFromDirectives(const LoopGraph *loopInfo, ParallelDirective *directive)
+{
+    SgForStmt *currentLoop = (SgForStmt*)loopInfo->loop;
+    for (auto &data : getAttributes<SgStatement*, SgStatement*>(currentLoop, set<int>{ SPF_ANALYSIS_DIR, SPF_PARALLEL_DIR, SPF_TRANSFORM_DIR }))
+    {
+        Statement *sData = new Statement(data);
+        fillPrivatesFromComment(sData, directive->privates);
+        fillReductionsFromComment(sData, directive->reduction, true);
+        fillReductionsFromComment(sData, directive->reductionLoc, true);
+        fillShadowAcrossFromComment(SHADOW_OP, sData, directive->shadowRenew);
+        fillShadowAcrossFromComment(ACROSS_OP, sData, directive->across);
+
+        map<pair<Symbol*, string>, Expression*> remotes;
+        fillRemoteFromComment(sData, remotes);
+        for (auto& elem : remotes)
+        {
+            SgSymbol* symb = OriginalSymbol(elem.first.first->GetOriginal());
+            auto uniqKey = getFromUniqTable(symb);
+            
+            directive->remoteAccess[make_pair(make_pair(elem.first.first->GetOriginal()->identifier(), getShortName(uniqKey)), elem.first.second)] = new Expression(elem.second->GetOriginal());
+        }        
+    }
+}
+
+void fillInfoFromDirective(Statement* parallel_on, DvmDirective& directive)
+{
+    Statement* sData = parallel_on;
+
+    fillReductionsFromComment(sData, directive.reduction, true, DVM_PARALLEL_ON_DIR);
+    fillReductionsFromComment(sData, directive.reductionLoc, true, DVM_PARALLEL_ON_DIR);
+    
+    fillShadowAcrossFromParallel(SHADOW_RENEW_OP, sData, directive.shadowRenew, directive.corners);
+    fillShadowAcrossFromParallel(ACROSS_OP, sData, directive.across, directive.corners);
+
+    //TODO:
+    /*map<pair<Symbol*, string>, Expression*> remotes;
+    fillRemoteFromComment(sData, remotes, false, DVM_PARALLEL_ON_DIR);
+    for (auto& elem : remotes)
+    {
+        SgSymbol* symb = OriginalSymbol(elem.first.first->GetOriginal());
+        auto uniqKey = getFromUniqTable(symb);
+
+        directive.remoteAccess[make_pair(make_pair(elem.first.first->GetOriginal()->identifier(), getShortName(uniqKey)), elem.first.second)] = new Expression(elem.second->GetOriginal());
+    }*/
+}
+
+int getCoverPropertyFromComment(Statement* stIn)
+{
+    if (stIn)
+    {
+        SgStatement* st = stIn->GetOriginal();
+        if (st->variant() == SPF_ANALYSIS_DIR)
+        {
+            SgExpression* exprList = st->expr(0);
+            while (exprList)
+            {
+                if (exprList->lhs() && exprList->lhs()->variant() == SPF_COVER_OP)
+                {
+                    auto value = exprList->lhs()->lhs();
+                    if (value->isInteger())
+                        return value->valueInteger();
+                    else
+                        return -1;
+                }
+                exprList = exprList->rhs();
+            }
+        }
+    }
+    return 0;
+}

src/DirectiveProcessing/directive_parser.h

@@ -0,0 +1,55 @@
+#pragma once
+
+#include <string>
+#include <set>
+#include <map>
+
+#include "../Utils/AstWrapper.h"
+#include "../GraphLoop/graph_loops.h"
+#include "../Distribution/DvmhDirective.h"
+
+struct DvmDirective
+{
+    std::set<Symbol*> corners;
+    std::vector<std::pair<std::pair<Symbol*, std::string>, std::vector<std::pair<int, int>>>> shadowRenew, across;
+    std::map<std::pair<std::pair<std::string, std::string>, std::string>, Expression*> remoteAccess;
+
+    std::map<std::string, std::set<Symbol*>> reduction;
+    std::map<std::string, std::set<std::tuple<Symbol*, Symbol*, int>>> reductionLoc;
+};
+
+bool isSPF_NoInline(Statement *stPrev);
+
+template<typename fillType>
+void fillPrivatesFromComment(Statement *st, std::set<fillType> &privates, int type = -1);
+
+template<typename fillType>
+void fillReductionsFromComment(Statement *st, std::map<std::string, std::set<fillType>> &reduction, bool moduleNameAdd = false, int type = SPF_ANALYSIS_DIR);
+
+template<typename fillType>
+void fillReductionsFromComment(Statement *st, std::map<std::string, std::set<std::tuple<fillType, fillType, int>>> &reduction, bool moduleNameAdd = false, int type = SPF_ANALYSIS_DIR);
+
+void fillParameterFromComment(Statement *st, std::vector<std::pair<Expression*, Expression*>> &assigns);
+
+template<typename fillType>
+void fillShadowAcrossFromComment(const int type, Statement *st, std::vector<std::pair<std::pair<fillType, std::string>, std::vector<std::pair<int, int>>>> &data);
+
+template<typename fillType>
+void fillShadowAcrossFromParallel(const int type, Statement *st, std::vector<std::pair<std::pair<fillType, std::string>, std::vector<std::pair<int, int>>>> &data, std::set<fillType> &corner);
+
+template<typename fillType>
+void fillRemoteFromComment(Statement *st, std::map<std::pair<fillType, std::string>, Expression*> &remote, bool isFull = false, int type = SPF_PARALLEL_DIR);
+
+void fillAcrossInfoFromDirectives(const LoopGraph *loopInfo, std::vector<std::pair<std::pair<std::string, std::string>, std::vector<std::pair<int, int>>>> &acrossInfo);
+void fillInfoFromDirectives(const LoopGraph *loopInfo, ParallelDirective *directive);
+void fillInfoFromDirective(Statement* parallel_on, DvmDirective& directive);
+
+void fillFissionPrivatesExpansionFromComment(Statement *stIn, std::vector<std::string> &vars);
+
+template<typename fillType>
+void fillShrinkFromComment(Statement *stIn, std::vector<std::pair<fillType, std::vector<int>>> &varDims);
+
+template<typename fillType>
+void fillCheckpointFromComment(Statement *stIn, std::map<int, Expression*> &clauses, std::set<fillType> &vars, std::set<fillType> &expt);
+
+int getCoverPropertyFromComment(Statement* stIn);

src/DirectiveProcessing/insert_directive.h

@@ -0,0 +1,63 @@
+#pragma once
+
+SgStatement* createStatFromExprs(const std::vector<Expression*>& exprs);
+
+void insertLoopTempalteDeclaration(SgFile* file, const DataDirective& dataDir,
+                                   const std::vector<std::string>& distrRules, const std::vector<std::vector<dist>>& distrRulesSt,
+                                   const DIST::Arrays<int>& allArrays,
+                                   const bool extractDir, const uint64_t regionId);
+
+void insertTempalteDeclarationToMainFile(SgFile* file, const DataDirective& dataDir,
+    const std::map<std::string, std::string>& templateDeclInIncludes,
+    const std::vector<std::string>& distrRules, const std::vector<std::vector<dist>>& distrRulesSt,
+    const DIST::Arrays<int>& allArrays,
+    const bool extractDir, const uint64_t regionId,
+    const std::set<std::string>& includedToThisFile);
+
+void insertDirectiveToFile(SgFile* file, const char* fin_name, const std::vector<Directive*>& toInsert, const bool extractDir, std::vector<Messages>& messagesForFile);
+
+void insertDistributionToFile(SgFile* file, const char* fin_name, const DataDirective& dataDir,
+    const std::set<std::string>& distrArrays, const std::vector<std::string>& distrRules,
+    const std::vector<std::vector<dist>>& distrRulesSt,
+    const std::vector<std::string>& alignRules,
+    const std::map<std::string, std::vector<LoopGraph*>>& loopGraph,
+    const DIST::Arrays<int>& allArrays,
+    DIST::GraphCSR<int, double, attrType>& reducedG,
+    std::map<std::string, std::map<int, std::set<std::string>>>& commentsToInclude,
+    std::map<std::string, std::string>& templateDeclInIncludes,
+    const bool extractDir, std::vector<Messages>& messagesForFile,
+    const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls,
+    const std::map<std::string, FuncInfo*>& funcsInFile,
+    const uint64_t regionId,
+    const std::set<std::string>& allFileNames);
+
+void insertShadowSpecToFile(SgFile* file, const char* fin_name, const std::set<std::string>& distrArrays,
+    DIST::GraphCSR<int, double, attrType>& reducedG,
+    std::map<std::string, std::map<int, std::set<std::string>>>& commentsToInclude,
+    const bool extractDir, std::vector<Messages>& messagesForFile,
+    const std::map<std::tuple<int, std::string, std::string>, std::pair<DIST::Array*, DIST::ArrayAccessInfo*>>& declaredArrays);
+
+void insertDistributionToFile(const char* origFile, const char* outFile, const std::map<int, std::set<std::string>>& commentsToInclude);
+void removeDvmDirectives(SgFile* file, const bool toComment);
+
+void insertDistributeDirsToParallelRegions(const std::vector<ParallelRegionLines>* currLines,
+    const std::vector<Statement*>& reDistrRulesBefore,
+    const std::vector<Statement*>& reDistrRulesAfter,
+    const std::vector<Statement*>& reAlignRules);
+
+void insertTemplateModuleUse(SgFile* file, const std::set<uint64_t>& regNum, const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls);
+void removeStatementsFromAllproject(const std::set<int>& variants);
+void correctTemplateModuleDeclaration(const std::string& folderName);
+
+void insertParallelDirs(SgFile* file, bool extract,
+                        std::vector<Directive*>& createdDirectives,
+                        std::vector<Messages>& messages,
+                        std::map<std::string, std::string>& templateDeclInIncludes,
+                        std::map<std::string, std::map<int, std::set<std::string>>>& commentsToInclude,
+                        const std::vector<FuncInfo*>& callGraph,
+                        const std::vector<ParallelRegion*>& parallelRegions,
+                        const std::map<std::string, std::vector<LoopGraph*>>& loopGraph,
+                        const std::set<std::string>& allFileNames,
+                        const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls,
+                        const std::map<std::tuple<int, std::string, std::string>, std::pair<DIST::Array*, DIST::ArrayAccessInfo*>>& declaredArrays,
+                        const std::map<DIST::Array*, std::tuple<int, std::string, std::string>>& tableOfUniqNamesByArray);

src/DirectiveProcessing/remote_access.h

@@ -0,0 +1,48 @@
+#pragma once
+
+struct RemoteRequest
+{
+    LoopGraph* position;
+    int dimNum;
+    REMOTE_TYPE value;
+    int currLine;
+    int maxDimSize;
+
+    explicit RemoteRequest(LoopGraph* position, int dimNum, REMOTE_TYPE value, int currLine, int maxDimSize) :
+        position(position), dimNum(dimNum), value(value), currLine(currLine), maxDimSize(maxDimSize)
+    { }
+};
+
+std::map<std::string, ArrayRefExp*>
+createRemoteInParallel(const std::pair<LoopGraph*, const ParallelDirective*> under_dvm_dir,
+                       const DIST::Arrays<int> &allArrays,
+                       const std::map<LoopGraph*, std::map<DIST::Array*, ArrayInfo*>> &loopInfo,
+                       DIST::GraphCSR<int, double, attrType> &reducedG,
+                       const DataDirective &data,
+                       const std::vector<int> &currVar,
+                       std::vector<Messages> &messages,
+                       const uint64_t regionId, 
+                       const std::map<DIST::Array*, std::set<DIST::Array*>> &arrayLinksByFuncCalls, 
+                       const std::map<std::string, FuncInfo*>& funcMap);
+
+void addRemoteLink(const LoopGraph* loop, const std::map<std::string, FuncInfo*>& funcMap, ArrayRefExp* expr, std::map<std::string, ArrayRefExp*>& uniqRemotes,
+                   const std::set<std::string>& remotesInParallel, std::set<ArrayRefExp*>& addedRemotes, const std::vector<std::string>& mapToLoop,
+                   std::vector<Messages>& messages, const int line, bool bindToLoopDistribution = true);
+
+ArrayRefExp* createRemoteLink(const LoopGraph* currLoop, const DIST::Array* forArray);
+
+std::vector<RemoteRequest> 
+     checkArrayRefInLoopForRemoteStatus(bool ifUnknownArrayAssignFound,
+                                        int sumMatched,
+                                        int numOfSubs,
+                                        int maxMatched,
+                                        int currLine,
+                                        DIST::Array* currArray,
+                                        std::vector<int>& wasFound,
+                                        ArrayRefExp* arrayRef,
+                                        std::map<LoopGraph*, std::map<DIST::Array*, ArrayInfo*>>& loopInfo,
+                                        const std::vector<int>& matchedToDim,
+                                        const std::map<int, LoopGraph*>& sortedLoopGraph,
+                                        const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls,
+                                        const ParallelRegion* region,
+                                        const std::vector<LoopGraph*>& parentLoops);

src/DirectiveProcessing/remote_access_base.cpp

@@ -0,0 +1,772 @@
+#include "../Utils/leak_detector.h"
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <cstdint>
+
+#include <string>
+#include <fstream>
+#include <iostream>
+#include <algorithm>
+
+#include <vector>
+#include <map>
+#include <set>
+#include <utility>
+#include <assert.h>
+
+#include "../Distribution/Arrays.h"
+#include "../GraphCall/graph_calls.h"
+#include "../GraphLoop/graph_loops.h"
+#include "../ParallelizationRegions/ParRegions.h"
+#include "remote_access.h"
+
+using std::vector;
+using std::pair;
+using std::tuple;
+using std::map;
+using std::set;
+using std::make_pair;
+using std::make_tuple;
+using std::get;
+using std::string;
+using std::wstring;
+
+#define DEB 0
+
+static set<string> fillRemotesInParallel(const ParallelDirective* dvm_dir)
+{
+    set<string> remotesInParallel;
+    for (auto& elem : dvm_dir->remoteAccess)
+        remotesInParallel.insert(elem.first.first.first + "(" + elem.first.second + ")");
+    return remotesInParallel;
+}
+
+static void createRemoteInParallel(const pair<LoopGraph*, const ParallelDirective*>& under_dvm_dir,
+                                   const set<DIST::Array*>& doneInLoops,
+                                   const map<string, FuncInfo*>& funcMap,
+                                   map<string, ArrayRefExp*>& uniqRemotes, vector<Messages>& messages,
+                                   const DataDirective& data, const vector<int>& currVar, const uint64_t regionId,
+                                   const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls)
+{
+    const set<DIST::Array*> usedArrays = under_dvm_dir.first->usedArrays;
+    const set<DIST::Array*> usedArraysWrite = under_dvm_dir.first->usedArraysWrite;
+    const set<string> remotesInParallel = fillRemotesInParallel(under_dvm_dir.second);
+    set<ArrayRefExp*> addedRemotes;
+
+    for (auto& usedArr : usedArrays)
+    {
+        if (doneInLoops.find(usedArr) == doneInLoops.end() && usedArraysWrite.find(usedArr) == usedArraysWrite.end())
+        {
+            set<DIST::Array*> realRefs;
+            getRealArrayRefs(usedArr, usedArr, realRefs, arrayLinksByFuncCalls);
+
+            bool isDistr = false;
+            
+            for (auto& realRef : realRefs)
+            {
+                auto templ = realRef->GetTemplateArray(regionId);
+                for (int z = 0; z < data.distrRules.size(); ++z)
+                {
+                    if (templ == data.distrRules[z].first)
+                    {
+                        for (auto& elem : data.distrRules[z].second[currVar[z]].distRule)
+                            if (elem == BLOCK)
+                                isDistr = true;
+                        break;
+                    }
+                }
+            }
+
+            if (isDistr)
+            {
+                vector<string> mapToLoop;
+                for (int z = 0; z < usedArr->GetDimSize(); ++z)
+                    mapToLoop.push_back("");
+
+                //TODO: find all array refs 
+                addRemoteLink(under_dvm_dir.first, funcMap, createRemoteLink(under_dvm_dir.first, usedArr), uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, -1, false);
+                __spf_print(DEB, "CRIP: %d, AFTER MAIN CHECK for arrray '%s'\n", __LINE__, usedArr->GetShortName().c_str());
+            }
+        }
+    }
+}
+
+static void createRemoteInParallel(const map<DIST::Array*, vector<ArrayOp>> remoteRegularReadsOfTopLoop,
+                                   const pair<LoopGraph*, const ParallelDirective*> under_dvm_dir,
+                                   const DIST::Arrays<int>& allArrays,
+                                   const map<LoopGraph*, map<DIST::Array*, ArrayInfo*>>& loopInfo,
+                                   DIST::GraphCSR<int, double, attrType>& reducedG,
+                                   const DataDirective& data,
+                                   const vector<int>& currVar,
+                                   map<string, ArrayRefExp*>& uniqRemotes,
+                                   vector<Messages>& messages,
+                                   const uint64_t regionId,
+                                   const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls,
+                                   set<DIST::Array*>& doneInLoops,
+                                   const map<string, FuncInfo*>& funcMap)
+{
+    if (!under_dvm_dir.first || !under_dvm_dir.second)
+        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+    __spf_print(DEB, "createRemoteInParallel %d, for loop %d\n", __LINE__, under_dvm_dir.first->lineNum);
+    set<ArrayRefExp*> addedRemotes;
+
+    auto it = loopInfo.find(under_dvm_dir.first);
+    if (it == loopInfo.end())
+    {
+        if (under_dvm_dir.first->perfectLoop > 1)
+        {
+            pair<LoopGraph*, const ParallelDirective*> nextDir = under_dvm_dir;
+            nextDir.first = under_dvm_dir.first->children[0];
+
+            createRemoteInParallel(remoteRegularReadsOfTopLoop, nextDir, allArrays, loopInfo, reducedG, data, currVar, uniqRemotes, messages, regionId, arrayLinksByFuncCalls, doneInLoops, funcMap);
+        }
+        return;
+    }
+
+    const map<DIST::Array*, ArrayInfo*>& currInfo = it->second;
+    const ParallelDirective* parDir = under_dvm_dir.second;
+
+    DIST::Array* arrayRefOnDir = parDir->arrayRef;
+    set<DIST::Array*> realRefArrayOnDir;
+    const set<string> remotesInParallel = fillRemotesInParallel(parDir);
+    const LoopGraph* currLoop = under_dvm_dir.first;
+    
+
+    if (!arrayRefOnDir->IsTemplate())
+    {
+        getRealArrayRefs(arrayRefOnDir, arrayRefOnDir, realRefArrayOnDir, arrayLinksByFuncCalls);
+        if (realRefArrayOnDir.size() != 1)
+        {
+            vector<vector<tuple<DIST::Array*, int, pair<int, int>>>> allRules(realRefArrayOnDir.size());
+            int tmpIdx = 0;
+            for (auto& array : realRefArrayOnDir)
+                reducedG.GetAlignRuleWithTemplate(array, allArrays, allRules[tmpIdx++], regionId);
+
+            if (!isAllRulesEqual(allRules))
+            {
+                __spf_print(1, "not supported yet\n");
+                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+            }
+            else
+                arrayRefOnDir = *(realRefArrayOnDir.begin());
+        }
+        else
+            arrayRefOnDir = *(realRefArrayOnDir.begin());
+    }
+
+    // for all array accesses in loop
+    for (auto& array : currInfo)
+    {
+        DIST::Array* arrayRef = array.first;
+        if (!arrayRef)
+            continue;
+        doneInLoops.insert(arrayRef);
+
+        const ArrayInfo* currArrayInfo = array.second;
+
+        set<DIST::Array*> realArrayRef;
+        getRealArrayRefs(arrayRef, arrayRef, realArrayRef, arrayLinksByFuncCalls);
+
+        for (auto& elem : realArrayRef)
+        {
+            arrayRef = elem;
+
+            // fill links between current array and array in parallel dir
+            vector<int> links;
+            if (arrayRef != arrayRefOnDir)
+                links = findLinksBetweenArrays(arrayRef, arrayRefOnDir, regionId);
+            else
+            {
+                links.resize(arrayRef->GetDimSize());
+                for (int k = 0; k < arrayRef->GetDimSize(); ++k)
+                    links[k] = k;
+            }
+
+            //fill info links with template
+            auto linksWithTempl = arrayRef->GetLinksWithTemplate(regionId);
+            auto alignRuleWithTempl = arrayRef->GetAlignRulesWithTemplate(regionId);
+
+            const DIST::Array* templArray = arrayRef->GetTemplateArray(regionId);
+            if (!templArray)
+                continue; // may be error?
+
+            DIST::Array* templRefOnDir = arrayRefOnDir;
+            if (!templRefOnDir->IsTemplate())
+                templRefOnDir = templRefOnDir->GetTemplateArray(regionId);
+
+            if (templArray != templRefOnDir) // different templates
+            {
+                doneInLoops.erase(array.first);
+                continue;
+            }
+
+
+            // fill distribute data variant
+            const DistrVariant* distrVar = NULL;
+            for (int k = 0; k < data.distrRules.size(); ++k)
+            {
+                if (data.distrRules[k].first == templArray)
+                {
+                    distrVar = &data.distrRules[k].second[currVar[k]];
+                    break;
+                }
+            }
+
+            if (!distrVar)
+                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+            // set new redistribute rule, if exist
+            const DistrVariant* newDistVar = currLoop->getRedistributeRule(templArray);
+            if (newDistVar)
+                distrVar = newDistVar;
+
+            set<string> parallelVars;
+            parallelVars.insert(parDir->parallel.begin(), parDir->parallel.end());
+
+            vector<string> mapToLoop;
+            for (int i = 0; i < links.size(); ++i)
+            {
+                if (links[i] != -1 && linksWithTempl[i] != -1)
+                {
+                    bool isCorrect = false;
+                    const bool isInParallel = parallelVars.find(parDir->on[links[i]].first) != parallelVars.end();
+                    if (distrVar->distRule[linksWithTempl[i]] == BLOCK)
+                    {
+                        if (parDir->on[links[i]].first != "*" && isInParallel)
+                            isCorrect = true;
+                        else
+                            isCorrect = false;
+                    }
+                    if (isCorrect)
+                        mapToLoop.push_back(parDir->on[links[i]].first);
+                    else
+                        mapToLoop.push_back("");
+                }
+                else
+                    mapToLoop.push_back("");
+            }
+
+            // main check 
+            for (int i = 0; i < links.size(); ++i)
+            {
+                bool needToCheck = false;
+                if (links[i] != -1 && linksWithTempl[i] != -1)
+                {
+                    const bool isInParallel = parallelVars.find(parDir->on[links[i]].first) != parallelVars.end();
+                    if (distrVar->distRule[linksWithTempl[i]] == BLOCK)
+                    {
+                        if (parDir->on[links[i]].first != "*" && !isInParallel)
+                            needToCheck = false;
+                        else
+                            needToCheck = true;
+                    }
+                }
+                else if (linksWithTempl[i] != -1)
+                {
+                    // if distributed and used in loop with out link with array on dir
+                    if (distrVar->distRule[linksWithTempl[i]] == BLOCK)
+                    {
+                        // check all unrecognized refs
+                        for (auto& unrec : currArrayInfo->arrayAccessUnrec)
+                        {
+                            if (addedRemotes.find(unrec.first) != addedRemotes.end())
+                                continue;
+                            else if (unrec.second.second[i] == REMOTE_TRUE)
+                            {
+                                addRemoteLink(currLoop, funcMap, unrec.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, unrec.second.first);
+                                __spf_print(DEB, "CRIP: %d, REMOTE_TRUE\n", __LINE__);
+                            }
+                        }
+
+                        // check all regular refs
+                        for (auto& reg : currArrayInfo->arrayAccess)
+                        {
+                            if (addedRemotes.find(reg.first) != addedRemotes.end())
+                                continue;
+
+                            if (arrayRef == arrayRefOnDir)
+                                continue;
+
+                            addRemoteLink(currLoop, funcMap, reg.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, reg.second.first);
+                            __spf_print(DEB, "CRIP: %d, IRREG_REFS\n", __LINE__);
+                        }
+                        continue;
+                    }
+                }
+
+                // check if current dimention is distributed
+                if (needToCheck)
+                {
+                    // check unregular acceses
+                    for (auto& unrec : currArrayInfo->arrayAccessUnrec)
+                    {
+                        if (addedRemotes.find(unrec.first) != addedRemotes.end())
+                            continue;
+
+                        if (unrec.second.second[i] == REMOTE_TRUE)
+                        {
+                            addRemoteLink(currLoop, funcMap, unrec.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, unrec.second.first);
+                            __spf_print(DEB, "CRIP: %d, IRREG_REFS && REMOTE_TRUE\n", __LINE__);
+                        }
+                    }
+
+                    // and check regular acceses
+                    for (auto& regAccess : currArrayInfo->arrayAccess)
+                    {
+                        if (addedRemotes.find(regAccess.first) != addedRemotes.end())
+                            continue;
+
+                        if (arrayRef == arrayRefOnDir)
+                            continue;
+
+                        //if has reads in more than one dim and it dims are dirstributed
+                        int countOfDimAcc = 0;
+                        for (int z = 0; z < linksWithTempl.size(); ++z)
+                        {
+                            bool distributed = false;
+                            if (linksWithTempl[z] != -1)
+                                distributed = (distrVar->distRule[linksWithTempl[z]] == BLOCK);
+
+                            countOfDimAcc += ((regAccess.second.second[z].coefficients.size() != 0) && distributed) ? 1 : 0;
+                        }
+
+                        if (countOfDimAcc > 1 ||
+                            countOfDimAcc == 1 && parDir->on[links[i]].first == "*")
+                        {
+                            addRemoteLink(currLoop, funcMap, regAccess.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, regAccess.second.first);
+                            __spf_print(DEB, "CRIP: %d ---\n", __LINE__);
+                            continue;
+                        }
+
+                        //if has distributed dim but loop maped to *
+                        if (parDir->on[links[i]].first == "*")
+                        {
+                            if (linksWithTempl[i] != -1)
+                                if (distrVar->distRule[linksWithTempl[i]] == BLOCK)
+                                {
+                                    addRemoteLink(currLoop, funcMap, regAccess.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, regAccess.second.first);
+                                    __spf_print(DEB, "CRIP: %d, NOT MAPPED\n", __LINE__);
+                                    continue;
+                                }
+                        }
+
+                        //if this array has no map rules to current array and this dim is distributed
+                        if (currLoop->directiveForLoop)
+                        {
+                            if (currLoop->directiveForLoop->on[links[i]].first != "*")
+                            {
+                                if (regAccess.second.second[i].coefficients.size() == 0)
+                                {
+                                    addRemoteLink(currLoop, funcMap, regAccess.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, regAccess.second.first);
+                                    __spf_print(DEB, "CRIP: %d, ----\n", __LINE__);
+                                    continue;
+                                }
+                            }
+                        }
+
+                        // check with loop alignment
+                        auto itRegRemote = remoteRegularReadsOfTopLoop.find(array.first);
+                        if (itRegRemote != remoteRegularReadsOfTopLoop.end())
+                        {
+                            bool wasAdd = false;
+                            for (auto& reads : regAccess.second.second[i].coefficients)
+                            {
+                                const pair<int, int>& currReadAcc = reads.first;
+                                for (auto& ref : itRegRemote->second[i].coefficients)
+                                {
+                                    if (ref.first == currReadAcc)
+                                    {
+                                        addRemoteLink(currLoop, funcMap, regAccess.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, regAccess.second.first);
+                                        __spf_print(DEB, "CRIP: %d, MISS\n", __LINE__);
+                                        wasAdd = true;
+                                        break;
+                                    }
+                                }
+
+                                if (wasAdd)
+                                    break;
+                            }
+                        }
+                    }
+                }
+            } // main check
+        }
+    }
+
+    if (currLoop->perfectLoop > 1)
+    {
+        pair<LoopGraph*, const ParallelDirective*> nextDir = under_dvm_dir;
+        nextDir.first = currLoop->children[0];
+
+        createRemoteInParallel(remoteRegularReadsOfTopLoop, nextDir, allArrays, loopInfo, reducedG, data, currVar, uniqRemotes, messages, regionId, arrayLinksByFuncCalls, doneInLoops, funcMap);
+    }
+}
+
+map<string, ArrayRefExp*> 
+createRemoteInParallel(const pair<LoopGraph*, const ParallelDirective*> under_dvm_dir,
+                       const DIST::Arrays<int>& allArrays,
+                       const map<LoopGraph*, map<DIST::Array*, ArrayInfo*>>& loopInfo,
+                       DIST::GraphCSR<int, double, attrType>& reducedG,
+                       const DataDirective& data,
+                       const vector<int>& currVar,
+                       vector<Messages>& messages,
+                       const uint64_t regionId,
+                       const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls,
+                       const map<string, FuncInfo*>& funcMap)
+{
+    map<string, ArrayRefExp*> uniqRemotes;
+    set<DIST::Array*> doneInLoops;
+
+    createRemoteInParallel(under_dvm_dir.first->remoteRegularReads, under_dvm_dir, allArrays, loopInfo, reducedG, data, currVar, uniqRemotes, messages, regionId, arrayLinksByFuncCalls, doneInLoops, funcMap);
+    createRemoteInParallel(under_dvm_dir, doneInLoops, funcMap, uniqRemotes, messages, data, currVar, regionId, arrayLinksByFuncCalls);
+
+    return uniqRemotes;
+}
+
+static void addInfoToMap(map<LoopGraph*, map<DIST::Array*, ArrayInfo*>>& loopInfo, LoopGraph* position, DIST::Array* array,
+                         ArrayRefExp* arrayRef, const int dimNum, const REMOTE_TYPE& value, const int currLine,
+                         const int maxDimSize, vector<RemoteRequest>& requests)
+{    
+    requests.push_back(RemoteRequest(position, dimNum, value, currLine, maxDimSize));
+
+    auto it = loopInfo.find(position);
+    if (loopInfo.end() == it)
+        it = loopInfo.insert(it, make_pair(position, map<DIST::Array*, ArrayInfo*>()));
+
+    auto it1 = it->second.find(array);
+    if (it1 == it->second.end())
+        it1 = it->second.insert(it1, make_pair(array, new ArrayInfo()));
+
+    auto it2 = it1->second->arrayAccessUnrec.find(arrayRef);
+    if (it2 == it1->second->arrayAccessUnrec.end())
+    {
+        it2 = it1->second->arrayAccessUnrec.insert(it2, make_pair(arrayRef, make_pair(currLine, vector<REMOTE_TYPE>())));
+
+        it2->second.second.resize(maxDimSize);
+        std::fill(it2->second.second.begin(), it2->second.second.end(), REMOTE_NONE);
+    }
+
+    if (dimNum == -1)
+    {
+        for (int z = 0; z < it2->second.second.size(); ++z)
+            it2->second.second[z] |= value;
+    }
+    else
+        it2->second.second[dimNum] |= value;
+
+    if (value == REMOTE_TRUE)
+        __spf_print(DEB, "RemoteAccess[%d]: true for dim %d and array %s, loop line %d\n", __LINE__, dimNum, array->GetShortName().c_str(), position->lineNum);
+}
+
+static pair<DIST::Array*, const DistrVariant*> 
+    getDistrVariant(const vector<pair<DIST::Array*, vector<DistrVariant>>>& distrRules,
+                    DIST::Array* forTempl, const vector<int>& currentVariant)
+{
+    pair<DIST::Array*, const DistrVariant*> currentVar;
+    for (int z1 = 0; z1 < currentVariant.size(); ++z1)
+    {
+        if (distrRules[z1].first == forTempl)
+        {
+            currentVar = make_pair(distrRules[z1].first, &distrRules[z1].second[currentVariant[z1]]);
+            break;
+        }
+    }
+    if (!currentVar.first)
+        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+    return currentVar;
+}
+
+// call this function for each array reference under loop(s)
+vector<RemoteRequest> 
+     checkArrayRefInLoopForRemoteStatus(bool ifUnknownArrayAssignFound,
+                                        int sumMatched, // sum of matched dimensions of 'arrayRef' to each 'parentLoops'
+                                        int numOfSubs, // number of subscriptions of 'arrayRef'
+                                        int maxMatched, // maximum numbe of matched dimensions of 'arrayRef' to each 'parentLoops'
+                                        int currLine, // current line in source code of 'arrayRef'
+                                        DIST::Array* currArray, // DIST::Array of 'arrayRef'
+                                        vector<int>& wasFoundForLoop, // size == 'parentLoops'.size(), init value -> 0, each position is sum of match dims count of 'arrayRef'
+                                        ArrayRefExp* arrayRef, // current expression of array ref under all 'parentLoops', 
+                                        map<LoopGraph*, map<DIST::Array*, ArrayInfo*>>& loopInfo,
+                                        const vector<int>& matchedToDim, // size == 'parentLoops'.size(), init value -> -1, each position is dimension of match 'arrayRef' for 'parentLoop'
+                                        const map<int, LoopGraph*>& sortedLoopGraph,                                        
+                                        const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls,
+                                        const ParallelRegion* region,
+                                        const vector<LoopGraph*>& parentLoops) // all loops which are associated with 'arrayRef'
+{
+    vector<RemoteRequest> requests;
+    if (sumMatched != parentLoops.size() && sumMatched == numOfSubs)
+    {
+        set<DIST::Array*> realArrayRefs;
+        getRealArrayRefs(currArray, currArray, realArrayRefs, arrayLinksByFuncCalls);
+
+        bool ok = true;
+        DIST::Array* templ = NULL;
+        vector<int> alignCoefs;
+        for (auto& real : realArrayRefs)
+        {
+            DIST::Array* curr = real->GetTemplateArray(region->GetId(), false);
+            alignCoefs = real->GetLinksWithTemplate(region->GetId());
+
+            if (templ == NULL)
+                templ = curr;
+            else if (templ != curr)
+                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+        }
+
+        string debOutStr = "";
+        char buf[256];
+        sprintf(buf, "RemoteAccess[%d]: check aligns for '%s'\n", __LINE__, currArray->GetShortName().c_str());
+        debOutStr += buf;
+
+        for (int z = 0; z < wasFoundForLoop.size(); ++z)
+        {
+            sprintf(buf, "RemoteAccess[%d]: check aligns %d == %d\n", __LINE__, z, wasFoundForLoop[z]);
+            debOutStr += buf;
+        }
+
+        for (int z = 0; z < matchedToDim.size(); ++z)
+        {
+            sprintf(buf, "RemoteAccess[%d]: matchedToDim[%d] = %d\n", __LINE__, z, matchedToDim[z]);
+            debOutStr += buf;
+        }
+
+        for (int l = 0; l < alignCoefs.size(); ++l)
+        {
+            sprintf(buf, "RemoteAccess[%d]: alignCoefs[%d] = %d\n", __LINE__, l, alignCoefs[l]);
+            debOutStr += buf;
+        }
+
+        //check array's alignment
+        for (int z = 0; z < wasFoundForLoop.size() && ok; ++z)
+        {
+            auto it = sortedLoopGraph.find(parentLoops[z]->lineNum);
+            if (it == sortedLoopGraph.end())
+                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+
+            if (!templ)
+                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+            const DataDirective& dataDirectives = region->GetDataDir();
+            const vector<int>& currentVariant = region->GetCurrentVariant();
+                        
+            pair<DIST::Array*, const DistrVariant*> currentVar = getDistrVariant(dataDirectives.distrRules, templ, currentVariant);
+
+            LoopGraph* loop = it->second;
+            if (!loop->directiveForLoop)
+            {
+                if (wasFoundForLoop[z])
+                {
+                    if (matchedToDim[z] != -1 && alignCoefs[matchedToDim[z]] != -1 && 
+                        currentVar.second->distRule[alignCoefs[matchedToDim[z]]] == distType::BLOCK)
+                    {
+                        ok = false;
+                        if (!ok)
+                        {
+                            __spf_print(DEB, "%s\n", debOutStr.c_str());
+                            __spf_print(DEB, "RemoteAccess[%d]: call addInfoMaps from aligns miss\n", __LINE__);
+                            __spf_print(DEB, "RemoteAccess[%d]: z = %d\n", __LINE__, z);
+                            addInfoToMap(loopInfo, parentLoops[z], currArray, arrayRef, matchedToDim[z], REMOTE_TRUE, currLine, numOfSubs, requests);
+                        }
+                    }
+                }
+                continue;
+            }
+
+            //apply redistribute
+            if (loop->getRedistributeRule(currentVar.first) != NULL)
+                currentVar.second = loop->getRedistributeRule(currentVar.first);
+
+            DIST::Array* loopT = loop->directiveForLoop->arrayRef;
+            sprintf(buf, "RemoteAccess[%d]: z = %d, array '%s'\n", __LINE__, z, loopT->GetShortName().c_str());
+            debOutStr += buf;
+
+            int dimToMap = -1;
+            for (int z1 = 0; z1 < loopT->GetDimSize(); ++z1)
+                if (loop->directiveForLoop->on[z1].first != "*")
+                    dimToMap = z1;
+
+            sprintf(buf, "RemoteAccess[%d]: z = %d, dimToMap = %d\n", __LINE__, z, dimToMap);
+            debOutStr += buf;
+            if (dimToMap != -1)
+            {
+                if (loopT != templ && !loopT->IsTemplate())
+                {
+                    sprintf(buf, "RemoteAccess[%d]: z = %d, false check !=\n", __LINE__, z);
+                    debOutStr += buf;
+
+                    DIST::Array* loopTempl = loopT->GetTemplateArray(region->GetId(), false);
+                    vector<int> loopAlignCoefs = loopT->GetLinksWithTemplate(region->GetId());
+
+                    if (loopTempl == NULL)
+                    {
+                        set<DIST::Array*> tmpSet;
+                        getRealArrayRefs(loopT, loopT, tmpSet, arrayLinksByFuncCalls);
+
+                        set<DIST::Array*> templates;
+                        for (auto& elem : tmpSet)
+                        {
+                            loopTempl = elem->GetTemplateArray(region->GetId(), false);
+                            loopAlignCoefs = elem->GetLinksWithTemplate(region->GetId());
+                            templates.insert(loopTempl);
+                        }
+
+                        if (templates.size() != 1)
+                            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+                    }
+
+                    if (loopTempl)
+                    {
+                        sprintf(buf, "RemoteAccess[%d]: z = %d, array '%s'\n", __LINE__, z, loopTempl->GetShortName().c_str());
+                        debOutStr += buf;
+                    }
+
+                    if (templ != loopTempl)
+                        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+                    if (loopAlignCoefs.size() == 0)
+                        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+                    if (loopAlignCoefs.size() <= dimToMap)
+                        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+                    if (loopAlignCoefs[dimToMap] == -1)
+                    {
+                        //TODO: need more tests for this case: prev bug => bugreport_1676461809
+                        /*if (loop->hasParalleDirectiveBefore())
+                            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+                        else*/
+                            continue;
+                    }
+                    else
+                        dimToMap = loopAlignCoefs[dimToMap];
+                }
+                else if (loopT != templ && loopT->IsTemplate())
+                {
+                    bool isDistr = false;
+                    for (auto& elem : currentVar.second->distRule)
+                        if (elem == BLOCK)
+                            isDistr = true;
+
+                    if (isDistr)
+                    {
+                        ok = false;
+                        __spf_print(DEB, "%s\n", debOutStr.c_str());
+                        __spf_print(DEB, "RemoteAccess[%d]: call addInfoMaps from template miss\n", __LINE__);
+                        __spf_print(DEB, "RemoteAccess[%d]: z = %d\n", __LINE__, z);
+                        addInfoToMap(loopInfo, parentLoops[z], currArray, arrayRef, matchedToDim[z], REMOTE_TRUE, currLine, numOfSubs, requests);                        
+                    }
+                    continue;
+                }
+
+                sprintf(buf, "RemoteAccess[%d]: ** z = %d, dimToMap = %d\n", __LINE__, z, dimToMap);
+                debOutStr += buf;
+
+                for (int z = 0; z < currentVar.second->distRule.size(); ++z)
+                {
+                    sprintf(buf, "RemoteAccess[%d]: distRule[%d] = %d\n", __LINE__, z, currentVar.second->distRule[z]);
+                    debOutStr += buf;
+                }
+
+                if (wasFoundForLoop[z])
+                {
+                    if (matchedToDim[z] != -1 && currentVar.second->distRule[alignCoefs[matchedToDim[z]]] == distType::BLOCK)
+                    {
+                        bool found = false;
+
+                        for (int l = 0; l < alignCoefs.size(); ++l)
+                        {
+                            if (alignCoefs[l] == dimToMap)
+                                found = true;
+                        }
+                        ok = found;
+
+                        if (!ok)
+                        {
+                            __spf_print(DEB, "%s\n", debOutStr.c_str());
+                            __spf_print(DEB, "RemoteAccess[%d]: call addInfoMaps from aligns miss\n", __LINE__);
+                            __spf_print(DEB, "RemoteAccess[%d]: z = %d\n", __LINE__, z);
+                            __spf_print(DEB, "RemoteAccess[%d]: dimToMap = %d\n", __LINE__, dimToMap);
+                            addInfoToMap(loopInfo, parentLoops[z], currArray, arrayRef, matchedToDim[z], REMOTE_TRUE, currLine, numOfSubs, requests);
+                        }
+                    }
+                }
+                else
+                {
+                    bool distrAny = false;
+                    bool distrAll = true;
+                    int dDim = -1;
+
+                    for (int idx = 0; idx < alignCoefs.size(); ++idx)
+                    {
+                        if (currentVar.second->distRule[idx] == distType::BLOCK)
+                        {
+                            dDim = idx;
+                            distrAny = true;
+                        }
+                        else
+                            distrAll = false;
+                    }
+
+
+                    if (matchedToDim[z] == -1 && distrAny && !distrAll)
+                    {
+                        ok = false;
+                        if (!ok)
+                        {
+                            __spf_print(DEB, "%s\n", debOutStr.c_str());
+                            __spf_print(DEB, "RemoteAccess[%d]: call addInfoMaps from aligns miss\n", __LINE__);
+                            __spf_print(DEB, "RemoteAccess[%d]: z = %d\n", __LINE__, z);
+                            __spf_print(DEB, "RemoteAccess[%d]: dimToMap = %d\n", __LINE__, dimToMap);
+                            addInfoToMap(loopInfo, parentLoops[z], currArray, arrayRef, dDim, REMOTE_TRUE, currLine, numOfSubs, requests);
+                        }
+                    }
+                }
+            }
+        }
+
+        if (ok && currArray->GetDimSize() > 1)
+            for (int z = 0; z < wasFoundForLoop.size(); ++z)
+                wasFoundForLoop[z] = 1;
+    }
+
+    if (ifUnknownArrayAssignFound)
+    {
+        if (sumMatched != numOfSubs ||
+            maxMatched != 1 ||
+            (sumMatched != parentLoops.size() && sumMatched != numOfSubs))
+        {
+            int local = 0;
+            bool hasLimits = false;
+            for (int i = 0; i < wasFoundForLoop.size(); ++i)
+            {
+                if (wasFoundForLoop[i] == 1)
+                {
+                    auto itLoop = sortedLoopGraph.find(parentLoops[i]->lineNum);
+                    if (itLoop == sortedLoopGraph.end())
+                        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+
+                    if (itLoop->second->hasLimitsToParallel())
+                        hasLimits = true;
+                }
+            }
+            for (int i = 0; i < wasFoundForLoop.size(); ++i)
+            {
+                if (wasFoundForLoop[i] != 1)
+                {
+                    for (int k = 0; k < numOfSubs; ++k)
+                    {
+                        if (hasLimits)
+                        {
+                            __spf_print(DEB, "RemoteAccess[%d]: call addInfoMaps from hasLimits\n", __LINE__);
+                            addInfoToMap(loopInfo, parentLoops[i], currArray, arrayRef, k, REMOTE_TRUE, currLine, numOfSubs, requests);
+                        }
+                    }
+                }
+            }
+        }
+    }
+    return requests;
+}

src/DirectiveProcessing/shadow.h

@@ -0,0 +1,150 @@
+#pragma once
+
+#include <map>
+#include <vector>
+#include <string>
+#include <set>
+
+struct FuncInfo;
+struct LoopGraph;
+struct ShadowNode;
+struct CommonBlock;
+
+typedef enum node : int { PARALLEL_DIR = 0, FUNCTION_CALL, PROCEDURE_CALL, STOP, START, END } nodeType;
+
+struct ShadowElement
+{
+    std::vector<std::pair<int, int>> bounds;
+    std::map<FuncInfo*, std::string> origNameByProc;
+    bool corner;
+
+    ShadowElement() { }
+    ShadowElement(const std::vector<std::pair<int, int>>& bounds,
+                  const std::pair<FuncInfo*, std::string>& origName, 
+                  bool corner) : bounds(bounds), corner(corner)
+    {
+        origNameByProc[origName.first] = origName.second;
+    }
+
+    bool operator==(const ShadowElement& left) const
+    {
+        return bounds == left.bounds && origNameByProc == left.origNameByProc && corner == left.corner;
+    }
+};
+
+struct NextNode
+{
+    ShadowNode* shNode;
+    std::set<DIST::Array*> writeTo;
+    bool isBackWard;
+    bool hasRealigns;
+
+    NextNode() { }
+
+    NextNode(ShadowNode* shNode, const std::set<DIST::Array*>& writeTo, bool hasRealigns = false)
+        : shNode(shNode), writeTo(writeTo), isBackWard(false), hasRealigns(hasRealigns)
+    { }
+
+    bool operator==(const NextNode& left) const
+    {
+        return (shNode == left.shNode) && (writeTo == left.writeTo);
+    }
+
+    bool operator<(const NextNode& left) const
+    {
+        return shNode < left.shNode;
+    }
+};
+
+struct PrevNode
+{
+    ShadowNode* shNode;
+    bool* isBackWard;
+    bool* hasRealigns;
+
+    PrevNode() { }
+    PrevNode(ShadowNode* shNode, bool* isBackWard = NULL, bool* hasRealigns = NULL)
+        : shNode(shNode), isBackWard(isBackWard), hasRealigns(hasRealigns)
+    { }
+
+    bool operator==(const PrevNode& left) const
+    {
+        return shNode == left.shNode;
+    }
+};
+
+struct ShadowNode
+{
+    void *info;
+    nodeType type;
+
+    //file and line
+    std::pair<FuncInfo*, int> location;
+
+    std::map<DIST::Array*, std::vector<ShadowElement>> shadows;
+    std::map<DIST::Array*, std::vector<ShadowElement>> newShadows;
+
+    std::vector<NextNode> next;
+    //std::vector<PrevNode> prev;
+
+    ShadowNode(void* info, nodeType type, FuncInfo* func, int line) : info(info), type(type), location(std::make_pair(func, line)) { }
+
+    void MoveShadow(const std::pair<DIST::Array*, std::vector<ShadowElement>>& shadowAdd)
+    {
+        for (auto& elem : shadowAdd.second)
+            newShadows[shadowAdd.first].push_back(elem);
+    }
+
+    void MergeNewShadowElements()
+    {
+        for (auto& elemByArray : newShadows)
+        {
+            ShadowElement newElems;
+            for (int z = 0; z < elemByArray.second.size(); ++z)
+            {
+                if (z == 0)
+                    newElems = elemByArray.second[0];
+                else
+                {
+                    newElems.corner |= elemByArray.second[z].corner;
+                    for (auto& fElem : elemByArray.second[z].origNameByProc)
+                        newElems.origNameByProc[fElem.first] = fElem.second;
+                    int k = 0;
+                    for (auto& bound : elemByArray.second[z].bounds)
+                    {
+                        newElems.bounds[k].first = std::max(newElems.bounds[k].first, bound.first);
+                        newElems.bounds[k].second = std::max(newElems.bounds[k].second, bound.second);
+                        ++k;
+                    }
+                }
+            }
+
+            if (elemByArray.second.size())
+            {
+                elemByArray.second.clear();
+                elemByArray.second.push_back(newElems);
+            }
+        }
+    }
+
+    void addNext(const NextNode& nextNode)
+    {
+        bool exist = false;
+        for (int z = 0; z < next.size() && !exist; ++z)
+            if (next[z] == nextNode)
+                exist = true;
+        
+        if (!exist)
+            next.push_back(nextNode);
+    }
+};
+
+void GroupShadow(const std::map<std::string, std::vector<FuncInfo*>>& funcs, 
+                 const std::map<std::string, std::vector<LoopGraph*>>& loops,
+                 const DIST::Arrays<int>& allArrays, 
+                 const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls,
+                 const std::map<std::string, CommonBlock*>& commonBlocks);
+
+void clearAllocatedShadowNodes();
+void devourShadowByRemote(void* file, const std::map<std::string, FuncInfo*>& allFuncs, const std::vector<LoopGraph*>& loops, const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls);
+void transformShadowIfFull(void* file, const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls);