SAPFOR/src/DirectiveProcessing/remote_access_base.cpp

#include "leak_detector.h"

#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <cstdint>

#include <string>
#include <fstream>
#include <iostream>
#include <algorithm>

#include <vector>
#include <map>
#include <set>
#include <utility>
#include <assert.h>

#include "../Distribution/Arrays.h"
#include "graph_calls.h"
#include "../GraphLoop/graph_loops.h"
#include "../ParallelizationRegions/ParRegions.h"
#include "remote_access.h"

using std::vector;
using std::pair;
using std::tuple;
using std::map;
using std::set;
using std::make_pair;
using std::make_tuple;
using std::get;
using std::string;
using std::wstring;

#define DEB 0

static set<string> fillRemotesInParallel(const ParallelDirective* dvm_dir)
{
    set<string> remotesInParallel;
    for (auto& elem : dvm_dir->remoteAccess)
        remotesInParallel.insert(elem.first.first.first + "(" + elem.first.second + ")");
    return remotesInParallel;
}

static void createRemoteInParallel(const pair<LoopGraph*, const ParallelDirective*>& under_dvm_dir,
                                   const set<DIST::Array*>& doneInLoops,
                                   const map<string, FuncInfo*>& funcMap,
                                   map<string, ArrayRefExp*>& uniqRemotes, vector<Messages>& messages,
                                   const DataDirective& data, const vector<int>& currVar, const uint64_t regionId,
                                   const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls)
{
    const set<DIST::Array*> usedArrays = under_dvm_dir.first->usedArrays;
    const set<DIST::Array*> usedArraysWrite = under_dvm_dir.first->usedArraysWrite;
    const set<string> remotesInParallel = fillRemotesInParallel(under_dvm_dir.second);
    set<ArrayRefExp*> addedRemotes;

    for (auto& usedArr : usedArrays)
    {
        if (doneInLoops.find(usedArr) == doneInLoops.end() && usedArraysWrite.find(usedArr) == usedArraysWrite.end())
        {
            set<DIST::Array*> realRefs;
            getRealArrayRefs(usedArr, usedArr, realRefs, arrayLinksByFuncCalls);

            bool isDistr = false;
            
            for (auto& realRef : realRefs)
            {
                auto templ = realRef->GetTemplateArray(regionId);
                for (int z = 0; z < data.distrRules.size(); ++z)
                {
                    if (templ == data.distrRules[z].first)
                    {
                        for (auto& elem : data.distrRules[z].second[currVar[z]].distRule)
                            if (elem == BLOCK)
                                isDistr = true;
                        break;
                    }
                }
            }

            if (isDistr)
            {
                vector<string> mapToLoop;
                for (int z = 0; z < usedArr->GetDimSize(); ++z)
                    mapToLoop.push_back("");

                //TODO: find all array refs 
                addRemoteLink(under_dvm_dir.first, funcMap, createRemoteLink(under_dvm_dir.first, usedArr), uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, -1, false);
                __spf_print(DEB, "CRIP: %d, AFTER MAIN CHECK for arrray '%s'\n", __LINE__, usedArr->GetShortName().c_str());
            }
        }
    }
}

static void createRemoteInParallel(const map<DIST::Array*, vector<ArrayOp>> remoteRegularReadsOfTopLoop,
                                   const pair<LoopGraph*, const ParallelDirective*> under_dvm_dir,
                                   const DIST::Arrays<int>& allArrays,
                                   const map<LoopGraph*, map<DIST::Array*, ArrayInfo*>>& loopInfo,
                                   DIST::GraphCSR<int, double, attrType>& reducedG,
                                   const DataDirective& data,
                                   const vector<int>& currVar,
                                   map<string, ArrayRefExp*>& uniqRemotes,
                                   vector<Messages>& messages,
                                   const uint64_t regionId,
                                   const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls,
                                   set<DIST::Array*>& doneInLoops,
                                   const map<string, FuncInfo*>& funcMap)
{
    if (!under_dvm_dir.first || !under_dvm_dir.second)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    __spf_print(DEB, "createRemoteInParallel %d, for loop %d\n", __LINE__, under_dvm_dir.first->lineNum);
    set<ArrayRefExp*> addedRemotes;

    auto it = loopInfo.find(under_dvm_dir.first);
    if (it == loopInfo.end())
    {
        if (under_dvm_dir.first->perfectLoop > 1)
        {
            pair<LoopGraph*, const ParallelDirective*> nextDir = under_dvm_dir;
            nextDir.first = under_dvm_dir.first->children[0];

            createRemoteInParallel(remoteRegularReadsOfTopLoop, nextDir, allArrays, loopInfo, reducedG, data, currVar, uniqRemotes, messages, regionId, arrayLinksByFuncCalls, doneInLoops, funcMap);
        }
        return;
    }

    const map<DIST::Array*, ArrayInfo*>& currInfo = it->second;
    const ParallelDirective* parDir = under_dvm_dir.second;

    DIST::Array* arrayRefOnDir = parDir->arrayRef;
    set<DIST::Array*> realRefArrayOnDir;
    const set<string> remotesInParallel = fillRemotesInParallel(parDir);
    const LoopGraph* currLoop = under_dvm_dir.first;
    

    if (!arrayRefOnDir->IsTemplate())
    {
        getRealArrayRefs(arrayRefOnDir, arrayRefOnDir, realRefArrayOnDir, arrayLinksByFuncCalls);
        if (realRefArrayOnDir.size() != 1)
        {
            vector<vector<tuple<DIST::Array*, int, pair<int, int>>>> allRules(realRefArrayOnDir.size());
            int tmpIdx = 0;
            for (auto& array : realRefArrayOnDir)
                reducedG.GetAlignRuleWithTemplate(array, allArrays, allRules[tmpIdx++], regionId);

            if (!isAllRulesEqual(allRules))
            {
                __spf_print(1, "not supported yet\n");
                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
            }
            else
                arrayRefOnDir = *(realRefArrayOnDir.begin());
        }
        else
            arrayRefOnDir = *(realRefArrayOnDir.begin());
    }

    // for all array accesses in loop
    for (auto& array : currInfo)
    {
        DIST::Array* arrayRef = array.first;
        if (!arrayRef)
            continue;
        doneInLoops.insert(arrayRef);

        const ArrayInfo* currArrayInfo = array.second;

        set<DIST::Array*> realArrayRef;
        getRealArrayRefs(arrayRef, arrayRef, realArrayRef, arrayLinksByFuncCalls);

        for (auto& elem : realArrayRef)
        {
            arrayRef = elem;

            // fill links between current array and array in parallel dir
            vector<int> links;
            if (arrayRef != arrayRefOnDir)
                links = findLinksBetweenArrays(arrayRef, arrayRefOnDir, regionId);
            else
            {
                links.resize(arrayRef->GetDimSize());
                for (int k = 0; k < arrayRef->GetDimSize(); ++k)
                    links[k] = k;
            }

            //fill info links with template
            auto linksWithTempl = arrayRef->GetLinksWithTemplate(regionId);
            auto alignRuleWithTempl = arrayRef->GetAlignRulesWithTemplate(regionId);

            const DIST::Array* templArray = arrayRef->GetTemplateArray(regionId);
            if (!templArray)
                continue; // may be error?

            DIST::Array* templRefOnDir = arrayRefOnDir;
            if (!templRefOnDir->IsTemplate())
                templRefOnDir = templRefOnDir->GetTemplateArray(regionId);

            if (templArray != templRefOnDir) // different templates
            {
                doneInLoops.erase(array.first);
                continue;
            }


            // fill distribute data variant
            const DistrVariant* distrVar = NULL;
            for (int k = 0; k < data.distrRules.size(); ++k)
            {
                if (data.distrRules[k].first == templArray)
                {
                    distrVar = &data.distrRules[k].second[currVar[k]];
                    break;
                }
            }

            if (!distrVar)
                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

            // set new redistribute rule, if exist
            const DistrVariant* newDistVar = currLoop->getRedistributeRule(templArray);
            if (newDistVar)
                distrVar = newDistVar;

            set<string> parallelVars;
            parallelVars.insert(parDir->parallel.begin(), parDir->parallel.end());

            vector<string> mapToLoop;
            for (int i = 0; i < links.size(); ++i)
            {
                if (links[i] != -1 && linksWithTempl[i] != -1)
                {
                    bool isCorrect = false;
                    const bool isInParallel = parallelVars.find(parDir->on[links[i]].first) != parallelVars.end();
                    if (distrVar->distRule[linksWithTempl[i]] == BLOCK)
                    {
                        if (parDir->on[links[i]].first != "*" && isInParallel)
                            isCorrect = true;
                        else
                            isCorrect = false;
                    }
                    if (isCorrect)
                        mapToLoop.push_back(parDir->on[links[i]].first);
                    else
                        mapToLoop.push_back("");
                }
                else
                    mapToLoop.push_back("");
            }

            // main check 
            for (int i = 0; i < links.size(); ++i)
            {
                bool needToCheck = false;
                if (links[i] != -1 && linksWithTempl[i] != -1)
                {
                    const bool isInParallel = parallelVars.find(parDir->on[links[i]].first) != parallelVars.end();
                    if (distrVar->distRule[linksWithTempl[i]] == BLOCK)
                    {
                        if (parDir->on[links[i]].first != "*" && !isInParallel)
                            needToCheck = false;
                        else
                            needToCheck = true;
                    }
                }
                else if (linksWithTempl[i] != -1)
                {
                    // if distributed and used in loop with out link with array on dir
                    if (distrVar->distRule[linksWithTempl[i]] == BLOCK)
                    {
                        // check all unrecognized refs
                        for (auto& unrec : currArrayInfo->arrayAccessUnrec)
                        {
                            if (addedRemotes.find(unrec.first) != addedRemotes.end())
                                continue;
                            else if (unrec.second.second[i] == REMOTE_TRUE)
                            {
                                addRemoteLink(currLoop, funcMap, unrec.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, unrec.second.first);
                                __spf_print(DEB, "CRIP: %d, REMOTE_TRUE\n", __LINE__);
                            }
                        }

                        // check all regular refs
                        for (auto& reg : currArrayInfo->arrayAccess)
                        {
                            if (addedRemotes.find(reg.first) != addedRemotes.end())
                                continue;

                            if (arrayRef == arrayRefOnDir)
                                continue;

                            addRemoteLink(currLoop, funcMap, reg.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, reg.second.first);
                            __spf_print(DEB, "CRIP: %d, IRREG_REFS\n", __LINE__);
                        }
                        continue;
                    }
                }

                // check if current dimention is distributed
                if (needToCheck)
                {
                    // check unregular acceses
                    for (auto& unrec : currArrayInfo->arrayAccessUnrec)
                    {
                        if (addedRemotes.find(unrec.first) != addedRemotes.end())
                            continue;

                        if (unrec.second.second[i] == REMOTE_TRUE)
                        {
                            addRemoteLink(currLoop, funcMap, unrec.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, unrec.second.first);
                            __spf_print(DEB, "CRIP: %d, IRREG_REFS && REMOTE_TRUE\n", __LINE__);
                        }
                    }

                    // and check regular acceses
                    for (auto& regAccess : currArrayInfo->arrayAccess)
                    {
                        if (addedRemotes.find(regAccess.first) != addedRemotes.end())
                            continue;

                        if (arrayRef == arrayRefOnDir)
                            continue;

                        //if has reads in more than one dim and it dims are dirstributed
                        int countOfDimAcc = 0;
                        for (int z = 0; z < linksWithTempl.size(); ++z)
                        {
                            bool distributed = false;
                            if (linksWithTempl[z] != -1)
                                distributed = (distrVar->distRule[linksWithTempl[z]] == BLOCK);

                            countOfDimAcc += ((regAccess.second.second[z].coefficients.size() != 0) && distributed) ? 1 : 0;
                        }

                        if (countOfDimAcc > 1 ||
                            countOfDimAcc == 1 && parDir->on[links[i]].first == "*")
                        {
                            addRemoteLink(currLoop, funcMap, regAccess.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, regAccess.second.first);
                            __spf_print(DEB, "CRIP: %d ---\n", __LINE__);
                            continue;
                        }

                        //if has distributed dim but loop maped to *
                        if (parDir->on[links[i]].first == "*")
                        {
                            if (linksWithTempl[i] != -1)
                                if (distrVar->distRule[linksWithTempl[i]] == BLOCK)
                                {
                                    addRemoteLink(currLoop, funcMap, regAccess.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, regAccess.second.first);
                                    __spf_print(DEB, "CRIP: %d, NOT MAPPED\n", __LINE__);
                                    continue;
                                }
                        }

                        //if this array has no map rules to current array and this dim is distributed
                        if (currLoop->directiveForLoop)
                        {
                            if (currLoop->directiveForLoop->on[links[i]].first != "*")
                            {
                                if (regAccess.second.second[i].coefficients.size() == 0)
                                {
                                    addRemoteLink(currLoop, funcMap, regAccess.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, regAccess.second.first);
                                    __spf_print(DEB, "CRIP: %d, ----\n", __LINE__);
                                    continue;
                                }
                            }
                        }

                        // check with loop alignment
                        auto itRegRemote = remoteRegularReadsOfTopLoop.find(array.first);
                        if (itRegRemote != remoteRegularReadsOfTopLoop.end())
                        {
                            bool wasAdd = false;
                            for (auto& reads : regAccess.second.second[i].coefficients)
                            {
                                const pair<int, int>& currReadAcc = reads.first;
                                for (auto& ref : itRegRemote->second[i].coefficients)
                                {
                                    if (ref.first == currReadAcc)
                                    {
                                        addRemoteLink(currLoop, funcMap, regAccess.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, regAccess.second.first);
                                        __spf_print(DEB, "CRIP: %d, MISS\n", __LINE__);
                                        wasAdd = true;
                                        break;
                                    }
                                }

                                if (wasAdd)
                                    break;
                            }
                        }
                    }
                }
            } // main check
        }
    }

    if (currLoop->perfectLoop > 1)
    {
        pair<LoopGraph*, const ParallelDirective*> nextDir = under_dvm_dir;
        nextDir.first = currLoop->children[0];

        createRemoteInParallel(remoteRegularReadsOfTopLoop, nextDir, allArrays, loopInfo, reducedG, data, currVar, uniqRemotes, messages, regionId, arrayLinksByFuncCalls, doneInLoops, funcMap);
    }
}

map<string, ArrayRefExp*> 
createRemoteInParallel(const pair<LoopGraph*, const ParallelDirective*> under_dvm_dir,
                       const DIST::Arrays<int>& allArrays,
                       const map<LoopGraph*, map<DIST::Array*, ArrayInfo*>>& loopInfo,
                       DIST::GraphCSR<int, double, attrType>& reducedG,
                       const DataDirective& data,
                       const vector<int>& currVar,
                       vector<Messages>& messages,
                       const uint64_t regionId,
                       const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls,
                       const map<string, FuncInfo*>& funcMap)
{
    map<string, ArrayRefExp*> uniqRemotes;
    set<DIST::Array*> doneInLoops;

    createRemoteInParallel(under_dvm_dir.first->remoteRegularReads, under_dvm_dir, allArrays, loopInfo, reducedG, data, currVar, uniqRemotes, messages, regionId, arrayLinksByFuncCalls, doneInLoops, funcMap);
    createRemoteInParallel(under_dvm_dir, doneInLoops, funcMap, uniqRemotes, messages, data, currVar, regionId, arrayLinksByFuncCalls);

    return uniqRemotes;
}

static void addInfoToMap(map<LoopGraph*, map<DIST::Array*, ArrayInfo*>>& loopInfo, LoopGraph* position, DIST::Array* array,
                         ArrayRefExp* arrayRef, const int dimNum, const REMOTE_TYPE& value, const int currLine,
                         const int maxDimSize, vector<RemoteRequest>& requests)
{    
    requests.push_back(RemoteRequest(position, dimNum, value, currLine, maxDimSize));

    auto it = loopInfo.find(position);
    if (loopInfo.end() == it)
        it = loopInfo.insert(it, make_pair(position, map<DIST::Array*, ArrayInfo*>()));

    auto it1 = it->second.find(array);
    if (it1 == it->second.end())
        it1 = it->second.insert(it1, make_pair(array, new ArrayInfo()));

    auto it2 = it1->second->arrayAccessUnrec.find(arrayRef);
    if (it2 == it1->second->arrayAccessUnrec.end())
    {
        it2 = it1->second->arrayAccessUnrec.insert(it2, make_pair(arrayRef, make_pair(currLine, vector<REMOTE_TYPE>())));

        it2->second.second.resize(maxDimSize);
        std::fill(it2->second.second.begin(), it2->second.second.end(), REMOTE_NONE);
    }

    if (dimNum == -1)
    {
        for (int z = 0; z < it2->second.second.size(); ++z)
            it2->second.second[z] |= value;
    }
    else
        it2->second.second[dimNum] |= value;

    if (value == REMOTE_TRUE)
        __spf_print(DEB, "RemoteAccess[%d]: true for dim %d and array %s, loop line %d\n", __LINE__, dimNum, array->GetShortName().c_str(), position->lineNum);
}

static pair<DIST::Array*, const DistrVariant*> 
    getDistrVariant(const vector<pair<DIST::Array*, vector<DistrVariant>>>& distrRules,
                    DIST::Array* forTempl, const vector<int>& currentVariant)
{
    pair<DIST::Array*, const DistrVariant*> currentVar;
    for (int z1 = 0; z1 < currentVariant.size(); ++z1)
    {
        if (distrRules[z1].first == forTempl)
        {
            currentVar = make_pair(distrRules[z1].first, &distrRules[z1].second[currentVariant[z1]]);
            break;
        }
    }
    if (!currentVar.first)
        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
    return currentVar;
}

// call this function for each array reference under loop(s)
vector<RemoteRequest> 
     checkArrayRefInLoopForRemoteStatus(bool ifUnknownArrayAssignFound,
                                        int sumMatched, // sum of matched dimensions of 'arrayRef' to each 'parentLoops'
                                        int numOfSubs, // number of subscriptions of 'arrayRef'
                                        int maxMatched, // maximum numbe of matched dimensions of 'arrayRef' to each 'parentLoops'
                                        int currLine, // current line in source code of 'arrayRef'
                                        DIST::Array* currArray, // DIST::Array of 'arrayRef'
                                        vector<int>& wasFoundForLoop, // size == 'parentLoops'.size(), init value -> 0, each position is sum of match dims count of 'arrayRef'
                                        ArrayRefExp* arrayRef, // current expression of array ref under all 'parentLoops', 
                                        map<LoopGraph*, map<DIST::Array*, ArrayInfo*>>& loopInfo,
                                        const vector<int>& matchedToDim, // size == 'parentLoops'.size(), init value -> -1, each position is dimension of match 'arrayRef' for 'parentLoop'
                                        const map<int, LoopGraph*>& sortedLoopGraph,                                        
                                        const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls,
                                        const ParallelRegion* region,
                                        const vector<LoopGraph*>& parentLoops) // all loops which are associated with 'arrayRef'
{
    vector<RemoteRequest> requests;
    if (sumMatched != parentLoops.size() && sumMatched == numOfSubs)
    {
        set<DIST::Array*> realArrayRefs;
        getRealArrayRefs(currArray, currArray, realArrayRefs, arrayLinksByFuncCalls);

        bool ok = true;
        DIST::Array* templ = NULL;
        vector<int> alignCoefs;
        for (auto& real : realArrayRefs)
        {
            DIST::Array* curr = real->GetTemplateArray(region->GetId(), false);
            alignCoefs = real->GetLinksWithTemplate(region->GetId());

            if (templ == NULL)
                templ = curr;
            else if (templ != curr)
                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
        }

        string debOutStr = "";
        char buf[256];
        sprintf(buf, "RemoteAccess[%d]: check aligns for '%s'\n", __LINE__, currArray->GetShortName().c_str());
        debOutStr += buf;

        for (int z = 0; z < wasFoundForLoop.size(); ++z)
        {
            sprintf(buf, "RemoteAccess[%d]: check aligns %d == %d\n", __LINE__, z, wasFoundForLoop[z]);
            debOutStr += buf;
        }

        for (int z = 0; z < matchedToDim.size(); ++z)
        {
            sprintf(buf, "RemoteAccess[%d]: matchedToDim[%d] = %d\n", __LINE__, z, matchedToDim[z]);
            debOutStr += buf;
        }

        for (int l = 0; l < alignCoefs.size(); ++l)
        {
            sprintf(buf, "RemoteAccess[%d]: alignCoefs[%d] = %d\n", __LINE__, l, alignCoefs[l]);
            debOutStr += buf;
        }

        //check array's alignment
        for (int z = 0; z < wasFoundForLoop.size() && ok; ++z)
        {
            auto it = sortedLoopGraph.find(parentLoops[z]->lineNum);
            if (it == sortedLoopGraph.end())
                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);


            if (!templ)
                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

            const DataDirective& dataDirectives = region->GetDataDir();
            const vector<int>& currentVariant = region->GetCurrentVariant();
                        
            pair<DIST::Array*, const DistrVariant*> currentVar = getDistrVariant(dataDirectives.distrRules, templ, currentVariant);

            LoopGraph* loop = it->second;
            if (!loop->directiveForLoop)
            {
                if (wasFoundForLoop[z])
                {
                    if (matchedToDim[z] != -1 && alignCoefs[matchedToDim[z]] != -1 && 
                        currentVar.second->distRule[alignCoefs[matchedToDim[z]]] == distType::BLOCK)
                    {
                        ok = false;
                        if (!ok)
                        {
                            __spf_print(DEB, "%s\n", debOutStr.c_str());
                            __spf_print(DEB, "RemoteAccess[%d]: call addInfoMaps from aligns miss\n", __LINE__);
                            __spf_print(DEB, "RemoteAccess[%d]: z = %d\n", __LINE__, z);
                            addInfoToMap(loopInfo, parentLoops[z], currArray, arrayRef, matchedToDim[z], REMOTE_TRUE, currLine, numOfSubs, requests);
                        }
                    }
                }
                continue;
            }

            //apply redistribute
            if (loop->getRedistributeRule(currentVar.first) != NULL)
                currentVar.second = loop->getRedistributeRule(currentVar.first);

            DIST::Array* loopT = loop->directiveForLoop->arrayRef;
            sprintf(buf, "RemoteAccess[%d]: z = %d, array '%s'\n", __LINE__, z, loopT->GetShortName().c_str());
            debOutStr += buf;

            int dimToMap = -1;
            for (int z1 = 0; z1 < loopT->GetDimSize(); ++z1)
                if (loop->directiveForLoop->on[z1].first != "*")
                    dimToMap = z1;

            sprintf(buf, "RemoteAccess[%d]: z = %d, dimToMap = %d\n", __LINE__, z, dimToMap);
            debOutStr += buf;
            if (dimToMap != -1)
            {
                if (loopT != templ && !loopT->IsTemplate())
                {
                    sprintf(buf, "RemoteAccess[%d]: z = %d, false check !=\n", __LINE__, z);
                    debOutStr += buf;

                    DIST::Array* loopTempl = loopT->GetTemplateArray(region->GetId(), false);
                    vector<int> loopAlignCoefs = loopT->GetLinksWithTemplate(region->GetId());

                    if (loopTempl == NULL)
                    {
                        set<DIST::Array*> tmpSet;
                        getRealArrayRefs(loopT, loopT, tmpSet, arrayLinksByFuncCalls);

                        set<DIST::Array*> templates;
                        for (auto& elem : tmpSet)
                        {
                            loopTempl = elem->GetTemplateArray(region->GetId(), false);
                            loopAlignCoefs = elem->GetLinksWithTemplate(region->GetId());
                            templates.insert(loopTempl);
                        }

                        if (templates.size() != 1)
                            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
                    }

                    if (loopTempl)
                    {
                        sprintf(buf, "RemoteAccess[%d]: z = %d, array '%s'\n", __LINE__, z, loopTempl->GetShortName().c_str());
                        debOutStr += buf;
                    }

                    if (templ != loopTempl)
                        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

                    if (loopAlignCoefs.size() == 0)
                        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

                    if (loopAlignCoefs.size() <= dimToMap)
                        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

                    if (loopAlignCoefs[dimToMap] == -1)
                    {
                        //TODO: need more tests for this case: prev bug => bugreport_1676461809
                        /*if (loop->hasParalleDirectiveBefore())
                            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
                        else*/
                            continue;
                    }
                    else
                        dimToMap = loopAlignCoefs[dimToMap];
                }
                else if (loopT != templ && loopT->IsTemplate())
                {
                    bool isDistr = false;
                    for (auto& elem : currentVar.second->distRule)
                        if (elem == BLOCK)
                            isDistr = true;

                    if (isDistr)
                    {
                        ok = false;
                        __spf_print(DEB, "%s\n", debOutStr.c_str());
                        __spf_print(DEB, "RemoteAccess[%d]: call addInfoMaps from template miss\n", __LINE__);
                        __spf_print(DEB, "RemoteAccess[%d]: z = %d\n", __LINE__, z);
                        addInfoToMap(loopInfo, parentLoops[z], currArray, arrayRef, matchedToDim[z], REMOTE_TRUE, currLine, numOfSubs, requests);                        
                    }
                    continue;
                }

                sprintf(buf, "RemoteAccess[%d]: ** z = %d, dimToMap = %d\n", __LINE__, z, dimToMap);
                debOutStr += buf;

                for (int z = 0; z < currentVar.second->distRule.size(); ++z)
                {
                    sprintf(buf, "RemoteAccess[%d]: distRule[%d] = %d\n", __LINE__, z, currentVar.second->distRule[z]);
                    debOutStr += buf;
                }

                if (wasFoundForLoop[z])
                {
                    if (matchedToDim[z] != -1 && currentVar.second->distRule[alignCoefs[matchedToDim[z]]] == distType::BLOCK)
                    {
                        bool found = false;

                        for (int l = 0; l < alignCoefs.size(); ++l)
                        {
                            if (alignCoefs[l] == dimToMap)
                                found = true;
                        }
                        ok = found;

                        if (!ok)
                        {
                            __spf_print(DEB, "%s\n", debOutStr.c_str());
                            __spf_print(DEB, "RemoteAccess[%d]: call addInfoMaps from aligns miss\n", __LINE__);
                            __spf_print(DEB, "RemoteAccess[%d]: z = %d\n", __LINE__, z);
                            __spf_print(DEB, "RemoteAccess[%d]: dimToMap = %d\n", __LINE__, dimToMap);
                            addInfoToMap(loopInfo, parentLoops[z], currArray, arrayRef, matchedToDim[z], REMOTE_TRUE, currLine, numOfSubs, requests);
                        }
                    }
                }
                else
                {
                    bool distrAny = false;
                    bool distrAll = true;
                    int dDim = -1;

                    for (int idx = 0; idx < alignCoefs.size(); ++idx)
                    {
                        if (currentVar.second->distRule[idx] == distType::BLOCK)
                        {
                            dDim = idx;
                            distrAny = true;
                        }
                        else
                            distrAll = false;
                    }


                    if (matchedToDim[z] == -1 && distrAny && !distrAll)
                    {
                        ok = false;
                        if (!ok)
                        {
                            __spf_print(DEB, "%s\n", debOutStr.c_str());
                            __spf_print(DEB, "RemoteAccess[%d]: call addInfoMaps from aligns miss\n", __LINE__);
                            __spf_print(DEB, "RemoteAccess[%d]: z = %d\n", __LINE__, z);
                            __spf_print(DEB, "RemoteAccess[%d]: dimToMap = %d\n", __LINE__, dimToMap);
                            addInfoToMap(loopInfo, parentLoops[z], currArray, arrayRef, dDim, REMOTE_TRUE, currLine, numOfSubs, requests);
                        }
                    }
                }
            }
        }

        if (ok && currArray->GetDimSize() > 1)
            for (int z = 0; z < wasFoundForLoop.size(); ++z)
                wasFoundForLoop[z] = 1;
    }

    if (ifUnknownArrayAssignFound)
    {
        if (sumMatched != numOfSubs ||
            maxMatched != 1 ||
            (sumMatched != parentLoops.size() && sumMatched != numOfSubs))
        {
            int local = 0;
            bool hasLimits = false;
            for (int i = 0; i < wasFoundForLoop.size(); ++i)
            {
                if (wasFoundForLoop[i] == 1)
                {
                    auto itLoop = sortedLoopGraph.find(parentLoops[i]->lineNum);
                    if (itLoop == sortedLoopGraph.end())
                        printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

                    if (itLoop->second->hasLimitsToParallel())
                        hasLimits = true;
                }
            }
            for (int i = 0; i < wasFoundForLoop.size(); ++i)
            {
                if (wasFoundForLoop[i] != 1)
                {
                    for (int k = 0; k < numOfSubs; ++k)
                    {
                        if (hasLimits)
                        {
                            __spf_print(DEB, "RemoteAccess[%d]: call addInfoMaps from hasLimits\n", __LINE__);
                            addInfoToMap(loopInfo, parentLoops[i], currArray, arrayRef, k, REMOTE_TRUE, currLine, numOfSubs, requests);
                        }
                    }
                }
            }
        }
    }
    return requests;
}