sapfor/experts/Sapfor_2017/CMakeLists.txt

@@ -193,6 +193,8 @@ set(TR_CONV _src/Transformations/convert_to_c.cpp
         _src/Transformations/convert_to_c.h)
 set(TR_IMPLICIT_NONE _src/Transformations/set_implicit_none.cpp
         _src/Transformations/set_implicit_none.h)
+set(TR_REPLACE_ARRAYS_IN_IO _src/Transformations/replace_dist_arrays_in_io.cpp
+        _src/Transformations/replace_dist_arrays_in_io.h)
 
 set(TRANSFORMS 
         ${TR_DEAD_CODE}
@@ -211,7 +213,8 @@ set(TRANSFORMS
         ${TR_PRIV_DEL}
         ${TR_CONV}
         ${TR_PRIV_DEL}
-        ${TR_IMPLICIT_NONE})
+        ${TR_IMPLICIT_NONE}
+        ${TR_REPLACE_ARRAYS_IN_IO})
         
 set(CFG _src/CFGraph/IR.cpp
         _src/CFGraph/IR.h
@@ -444,6 +447,7 @@ source_group (Transformations\\RenameSymbols FILES ${RENAME_SYMBOLS})
 source_group (Transformations\\GlobalVariables FILES ${TR_GV})
 source_group (Transformations\\ConvertToC FILES ${TR_CONV})
 source_group (Transformations\\SetImplicitNone FILES ${TR_IMPLICIT_NONE})
+source_group (Transformations\\ReplaceArraysInIO FILES ${TR_REPLACE_ARRAYS_IN_IO})
 
 
 source_group (CreateIntervals FILES ${CREATE_INTER_T})

sapfor/experts/Sapfor_2017/_src/ParallelizationRegions/resolve_par_reg_conflicts.cpp

@@ -958,7 +958,7 @@ static bool replaceCommonArray(const string &fileName,
     return false;
 }
 
-static pair<SgSymbol*, SgSymbol*> copyArray(const pair<string, int> &place,
+pair<SgSymbol*, SgSymbol*> copyArray(const pair<string, int> &place,
                                      const DIST::Array *array,
                                      const vector<ParallelRegionLines>& lines,
                                      const string &suffix, 

sapfor/experts/Sapfor_2017/_src/ParallelizationRegions/resolve_par_reg_conflicts.h

@@ -14,3 +14,11 @@ int printCheckRegions(const char *fileName, const std::vector<ParallelRegion*> &
 bool checkRegionsResolving(const std::vector<ParallelRegion*> &regions, const std::map<std::string, std::vector<FuncInfo*>> &allFuncInfo, const std::map<std::string, CommonBlock*> &commonBlocks, std::map<std::string, std::vector<Messages>> &SPF_messages, bool sharedMemoryParallelization);
 int resolveParRegions(std::vector<ParallelRegion*>& regions, const std::map<std::string, std::vector<FuncInfo*>>& allFuncInfo, std::map<std::string, std::vector<Messages>>& SPF_messages, bool sharedMemoryParallelization, std::map<std::string, std::map<int, std::set<std::string>>>& copyDecls);
 void insertRealignsBeforeFragments(ParallelRegion* reg, SgFile* file, const std::set<DIST::Array*>& distrArrays, const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls);
+
+std::pair<SgSymbol*, SgSymbol*> copyArray(const std::pair<std::string, int>& place,
+                                          const DIST::Array* array,
+                                          const std::vector<ParallelRegionLines>& lines,
+                                          const std::string& suffix,
+                                          std::string& filename,
+                                          std::map<std::string, std::map<int, std::set<std::string>>>& newDeclsToInclude,
+                                          std::map<std::string, std::map<int, std::set<std::string>>>& copied);

sapfor/experts/Sapfor_2017/_src/Sapfor.cpp

@@ -24,6 +24,8 @@
 #include "ParallelizationRegions/resolve_par_reg_conflicts.h"
 #include "ParallelizationRegions/expand_extract_reg.h"
 
+#include "Transformations/replace_dist_arrays_in_io.h"
+
 #include "Distribution/Distribution.h"
 #include "Distribution/GraphCSR.h"
 #include "Distribution/Arrays.h"
@@ -1567,6 +1569,10 @@ static bool runAnalysis(SgProject &project, const int curr_regime, const bool ne
         if (error)
             internalExit = 1;
     }
+    else if (curr_regime == REMOVE_DIST_ARRAYS_FROM_IO)
+    {
+        replaceDistributedArraysInIO(parallelRegions, allFuncInfo, SPF_messages, newCopyDeclToIncl);
+    }
     else if (curr_regime == LOOP_GRAPH)
     {
         if (keepFiles)
@@ -2333,6 +2339,7 @@ void runPass(const int curr_regime, const char *proj_name, const char *folderNam
         findFunctionsToInclude(true);
         break;
         // all these cases run UNPARSE_FILE after
+    case REMOVE_DIST_ARRAYS_FROM_IO:
     case RENAME_SYMBOLS:
     case RESOLVE_PAR_REGIONS:
     case CREATE_PARALLEL_REGIONS:

sapfor/experts/Sapfor_2017/_src/Sapfor.h

@@ -87,6 +87,8 @@ enum passes {
     REMOVE_DVM_INTERVALS,
     VERIFY_DVM_DIRS,
 
+    REMOVE_DIST_ARRAYS_FROM_IO,
+
     SUBST_EXPR,
     SUBST_EXPR_RD,
     REVERT_SUBST_EXPR,
@@ -260,6 +262,7 @@ static void setPassValues()
     passNames[INSERT_INCLUDES] = "INSERT_INCLUDES";
     passNames[REMOVE_DVM_DIRS] = "REMOVE_DVM_DIRS";
     passNames[VERIFY_DVM_DIRS] = "VERIFY_DVM_DIRS";
+    passNames[REMOVE_DIST_ARRAYS_FROM_IO] = "REMOVE_DIST_ARRAYS_FROM_IO";
     passNames[SUBST_EXPR] = "SUBST_EXPR";
     passNames[SUBST_EXPR_RD] = "SUBST_EXPR_RD";
     passNames[CALL_GRAPH2] = "CALL_GRAPH2";

sapfor/experts/Sapfor_2017/_src/Transformations/replace_dist_arrays_in_io.cpp

@@ -0,0 +1,478 @@
+#include "replace_dist_arrays_in_io.h"
+
+#include "../ParallelizationRegions/resolve_par_reg_conflicts.h"
+
+#include <string>
+#include <map>
+#include <set>
+
+using std::map;
+using std::set;
+using std::string;
+using std::vector;
+using std::to_string;
+using std::make_pair;
+
+#define DEBUG_TRACE 0
+
+static void findArrays(SgExpression* exp, set<SgSymbol*>& arrays)
+{
+    if (exp)
+    {
+        if (isSgArrayRefExp(exp))
+            arrays.insert(exp->symbol());
+
+        findArrays(exp->lhs(), arrays);
+        findArrays(exp->rhs(), arrays);
+    }
+}
+
+static void populateDistributedIoArrays(map<DIST::Array*, set<SgStatement*>>& arrays, SgStatement* stat)
+{
+    auto var = stat->variant();
+
+    if (var != READ_STAT && var != PRINT_STAT && var != WRITE_STAT)
+        return;
+
+    // check if such IO allowed in dvm: 
+    // list should consist only of single array and format string should be *
+
+    bool need_replace = false;
+
+    SgExpression* ioList = stat->expr(0);
+
+    if (!ioList)
+        return;
+
+    if (ioList->variant() != EXPR_LIST)
+        return;
+
+    if (ioList->rhs() == NULL)
+    {
+        SgExpression* arg = ioList->lhs();
+        if (!arg)
+            return;
+
+        if (arg->variant() != ARRAY_REF)
+            return;
+
+        if (arg->lhs())
+            need_replace = true;
+    }
+    else
+    {
+        need_replace = true;
+    }
+
+    if (!need_replace)
+    {
+        switch (var)
+        {
+        case PRINT_STAT:
+        {
+            SgExpression* fmt = stat->expr(1);
+            if (!fmt || fmt->variant() != SPEC_PAIR || fmt->lhs()->variant() != KEYWORD_VAL)
+            {
+                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+                return;
+            }
+
+            if (fmt->rhs()->variant() != KEYWORD_VAL || fmt->rhs()->sunparse() != "*")
+                need_replace = true;
+
+
+            break;
+        }
+        case READ_STAT:
+        case WRITE_STAT:
+        {
+
+            SgExpression* spec = stat->expr(1);
+            __spf_print(DEBUG_TRACE, "[%d: %s (%d)]\n", 2000, spec->rhs()->unparse(), spec->rhs()->variant());
+            if (!spec || spec->variant() != EXPR_LIST ||
+                spec->lhs()->variant() != SPEC_PAIR ||
+                !spec->rhs() || !spec->rhs()->lhs() || spec->rhs()->lhs()->variant() != SPEC_PAIR)
+            {
+                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+                return;
+            }
+
+            SgExpression* unit_val = spec->lhs()->rhs(), * fmt_val = spec->rhs()->lhs()->rhs();
+
+            if (unit_val->variant() != KEYWORD_VAL || unit_val->sunparse() != "*" ||
+                fmt_val->variant() != KEYWORD_VAL || fmt_val->sunparse() != "*")
+                need_replace = true;
+
+            break;
+        }
+        default:
+            break;
+        }
+    }
+
+    if (!need_replace)
+        return;
+
+    set<SgSymbol*> found_arrays;
+
+    for (int i = 0; i < 3; i++)
+        findArrays(stat->expr(i), found_arrays);
+
+    for (auto* by_symb : found_arrays)
+    {
+        string array_name = string(by_symb->identifier());
+        DIST::Array* array_p = getArrayFromDeclarated(declaratedInStmt(by_symb), array_name);
+        if (array_p && array_p->GetDistributeFlagVal() == Distribution::distFlag::DISTR && arrays[array_p].insert(stat).second)
+            __spf_print(DEBUG_TRACE, "[%d]: add array %s\n", stat->lineNumber(), array_p->GetName().c_str());
+    }
+
+    __spf_print(DEBUG_TRACE, "[replace]\n");
+}
+
+static void replaceArrayRec(SgSymbol* arr, SgSymbol* replace_by, SgExpression* exp, bool& has_read, bool& has_write, bool from_read, bool from_write)
+{
+    if (!exp)
+        return;
+
+    if (exp->symbol() && strcmp(exp->symbol()->identifier(), arr->identifier()) == 0)
+    {
+        has_read |= from_read;
+        has_write |= from_write;
+        exp->setSymbol(replace_by);
+    }
+
+    switch (exp->variant())
+    {
+        case FUNC_CALL:
+        {
+            replaceArrayRec(arr, replace_by, exp->rhs(), has_read, has_write, true, false);
+            replaceArrayRec(arr, replace_by, exp->lhs(), has_read, has_write, true, true);
+            break;
+        }
+        case EXPR_LIST:
+        {
+            replaceArrayRec(arr, replace_by, exp->lhs(), has_read, has_write, from_read, from_write);
+            replaceArrayRec(arr, replace_by, exp->rhs(), has_read, has_write, from_read, from_write);
+            break;
+        }
+        default:
+        {
+            replaceArrayRec(arr, replace_by, exp->lhs(), has_read, has_write, true, false);
+            replaceArrayRec(arr, replace_by, exp->rhs(), has_read, has_write, true, false);
+            break;
+        }
+    }
+}
+
+static void replaceArrayRec(SgSymbol* arr, SgSymbol* replace_by, SgStatement* st, bool& has_read, bool& has_write)
+{
+    if (!st)
+        return;
+
+    switch (st->variant())
+    {
+        case ASSIGN_STAT:
+        case READ_STAT:
+        {
+            replaceArrayRec(arr, replace_by, st->expr(0), has_read, has_write, false, true);
+            replaceArrayRec(arr, replace_by, st->expr(1), has_read, has_write, true, false);
+            break;
+        }
+        case PROC_STAT:
+        case FUNC_STAT:
+        {
+            replaceArrayRec(arr, replace_by, st->expr(0), has_read, has_write, true, false);
+            replaceArrayRec(arr, replace_by, st->expr(1), has_read, has_write, true, true);
+            break;
+        }
+        default:
+        {
+            for (int i = 0; i < 3; i++)
+                replaceArrayRec(arr, replace_by, st->expr(i), has_read, has_write, true, false);
+
+            break;
+        }
+    }
+}
+
+static void copyArrayBetweenStatements(SgSymbol* replace_symb, SgSymbol* replace_by, SgStatement* start, SgStatement* last)
+{
+    while (start->lexNext() && !isSgExecutableStatement(start->lexNext()))
+        start = start->lexNext();
+
+    auto* stop = last->lexNext();
+
+    bool has_read = false, has_write = false;
+
+    for (auto* st = start; st != stop; st = st->lexNext())
+        replaceArrayRec(replace_symb, replace_by, st, has_read, has_write);
+
+
+    if (has_read)
+    {
+        // A_copy = A
+        SgAssignStmt* assign = new SgAssignStmt(*new SgArrayRefExp(*replace_by), *new SgArrayRefExp(*replace_symb));
+        assign->setlineNumber(getNextNegativeLineNumber()); // before region
+        auto* parent = start->controlParent();
+        if (parent && parent->lastNodeOfStmt() == start)
+            parent = parent->controlParent();
+
+        start->insertStmtAfter(*assign, *parent);
+    }
+
+    if (has_write)
+    {
+        // A = A_reg
+        SgAssignStmt* assign = new SgAssignStmt(*new SgArrayRefExp(*replace_symb), *new SgArrayRefExp(*replace_by));
+        //TODO: bug with insertion
+        //assign->setlineNumber(getNextNegativeLineNumber()); // after region
+        last->insertStmtBefore(*assign, *(last->controlParent()));
+    }
+}
+
+static void replaceArrayInFragment(DIST::Array* arr, const set<SgStatement*> usages, SgSymbol* replace_by, SgStatement* start, SgStatement* last, const string& filename)
+{
+    while (start->lexNext() && !isSgExecutableStatement(start->lexNext()))
+        start = start->lexNext();
+
+    auto* replace_symb = arr->GetDeclSymbol();
+
+    set<SgStatement*> not_opened, not_closed, copied;
+
+    for (auto* it = start; it; it = it->controlParent())
+        not_opened.insert(it);
+
+    for (auto* it = last; it; it = it->controlParent())
+        not_closed.insert(it);
+
+    for (auto* io_stmt : usages)
+    {
+        bool already_copied = false;
+        SgStatement* copy_scope = NULL;
+
+        for (auto* par = io_stmt; par; par = par->controlParent())
+        {
+            if (copied.find(par) != copied.end())
+            {
+                already_copied = true;
+                break;
+            }
+            else if (not_opened.find(par) != not_opened.end() || not_closed.find(par) != not_closed.end())
+            {
+                copy_scope = par;
+                break;
+            }
+        }
+
+
+        if (already_copied)
+            continue;
+
+        auto* scope_start = copy_scope, * scope_end = copy_scope->lastNodeOfStmt();
+        __spf_print(DEBUG_TRACE, "[scope to copy] %d\n", copy_scope->lineNumber());
+
+        if (not_opened.find(copy_scope) != not_opened.end())
+        {
+            auto* from = start->lastNodeOfStmt() ? start->lastNodeOfStmt() : start;
+            for (auto* st = from; st; st = st->controlParent())
+            {
+                __spf_print(DEBUG_TRACE, "[find start of parent %d] %d\n", copy_scope->lineNumber(), st->lineNumber());
+
+                if (st->controlParent() == copy_scope)
+                {
+                    scope_start = st->lastNodeOfStmt() ? st->lastNodeOfStmt() : st;
+                    break;
+                }
+            }
+        }
+
+        if (not_closed.find(copy_scope) != not_closed.end())
+        {
+            for (auto* st = last; st; st = st->controlParent())
+            {
+                __spf_print(DEBUG_TRACE, "[find end of parent %d] %d\n", copy_scope->lineNumber(), st->lineNumber());
+
+                if (st->controlParent() == copy_scope)
+                {
+                    scope_end = st;
+                    break;
+                }
+            }
+        }
+
+        copyArrayBetweenStatements(replace_symb, replace_by, scope_start, scope_end);
+        __spf_print(DEBUG_TRACE, "[copy %s] [%d, %d]\n", arr->GetName().c_str(), scope_start->lineNumber(), scope_end->lineNumber());
+        copied.insert(copy_scope);
+    }
+}
+
+static bool ioReginBound(SgStatement* stat, SgStatement* last_io_bound)
+{
+    auto var = stat->variant();
+
+    if (var == PROC_STAT || var == FUNC_STAT || var == PROG_HEDR || var == FUNC_HEDR || var == PROC_HEDR || var == FUNC_STAT || var == FOR_NODE || var == LOOP_NODE)
+        return true;
+
+    if (last_io_bound && last_io_bound->lastNodeOfStmt() && last_io_bound->lastNodeOfStmt() == stat)
+        return true;
+
+    int parent_var;
+
+    if (var == CONTROL_END &&
+        ((parent_var = stat->controlParent()->variant()) == PROG_HEDR ||
+            parent_var == PROC_HEDR || parent_var == FUNC_HEDR))
+    {
+        return true;
+    }
+
+    return false;
+}
+
+void replaceDistributedArraysInIO(vector<ParallelRegion*>& regions,
+                                  const map<string, vector<FuncInfo*>>& allFuncInfo,
+                                  map<string, vector<Messages>>& SPF_messages,
+                                  map<string, map<int, set<string>>>& newDeclsToInclude)
+{
+    map<DIST::Array*, SgSymbol*> created_copies;
+    map<string, map<int, set<string>>> copied;
+
+    for (auto& region : regions)
+    {
+        __spf_print(DEBUG_TRACE, "[%s]: enter region\n", region->GetName().c_str());
+
+        for (auto& linesByFile : region->GetAllLinesToModify())
+        {
+            const auto& filename = linesByFile.first;
+
+            if (SgFile::switchToFile(filename) < 0) {
+                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+                return;
+            }
+
+            for (auto& lines : linesByFile.second) {
+                __spf_print(DEBUG_TRACE, "[fragment] %s: %d:%d %d\n", filename.c_str(), lines.lines.first,
+                    lines.lines.second, lines.isImplicit());
+
+                SgStatement* curr_stmt, * end;
+
+                if (lines.isImplicit())
+                {
+
+                    curr_stmt = current_file->SgStatementAtLine(lines.lines.first);
+                    end = current_file->SgStatementAtLine(lines.lines.second);
+
+                    if (end)
+                        end = end->lexNext();
+                }
+                else
+                {
+                    curr_stmt = lines.stats.first->GetOriginal();
+                    end = lines.stats.second->GetOriginal()->lexNext();
+                }
+
+                map<DIST::Array*, set<SgStatement*>> need_replace;
+                SgStatement* last_io_bound = NULL;
+
+
+                while (curr_stmt != end)
+                {
+                    if (!curr_stmt)
+                        break;
+
+                    auto var = curr_stmt->variant();
+
+
+                    if (var == PROC_HEDR || var == PROG_HEDR || var == FUNC_HEDR)
+                    {
+                        curr_stmt = curr_stmt->lexNext();
+                        while (curr_stmt && !isSgExecutableStatement(curr_stmt))
+                        {
+                            last_io_bound = curr_stmt;
+                            curr_stmt = curr_stmt->lexNext();
+                        }
+
+                        if (!curr_stmt)
+                            break;
+                    }
+
+                    if (ioReginBound(curr_stmt, last_io_bound))
+                    {
+                        if (last_io_bound)
+                        {
+                            __spf_print(DEBUG_TRACE, "[io region] [%d, %d]\n", last_io_bound->lineNumber(), curr_stmt->lineNumber());
+
+                            for (const auto& p : need_replace)
+                            {
+                                auto it = created_copies.find(p.first);
+                                if (it != created_copies.end())
+                                    replaceArrayInFragment(p.first, p.second, it->second, last_io_bound, curr_stmt, filename);
+                                else
+                                {
+                                    printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+                                    return;
+                                }
+                            }
+                        }
+
+                        need_replace.clear();
+                        last_io_bound = curr_stmt;
+                    }
+
+                    __spf_print(DEBUG_TRACE, "[line] %d (%d)\n", curr_stmt->lineNumber(), curr_stmt->variant());
+
+                    for (int i = 0; i < 3; i++)
+                    {
+                        if (curr_stmt->expr(i))
+                        {
+                            __spf_print(DEBUG_TRACE, "[%d: %s (%d)]\n", i, curr_stmt->expr(i)->unparse(), curr_stmt->expr(i)->variant());
+                        }
+                    }
+
+                    populateDistributedIoArrays(need_replace, curr_stmt);
+
+                    for (const auto& by_array_to_copy : need_replace)
+                    {
+                        auto* array_to_copy = by_array_to_copy.first;
+                        auto it = created_copies.find(array_to_copy);
+
+                        if (it == created_copies.end())
+                        {
+                            bool fromModule = (array_to_copy->GetLocation().first == DIST::l_MODULE);
+                            const string locationName = array_to_copy->GetLocation().second;
+
+                            auto place = *array_to_copy->GetDeclInfo().begin();
+                            string fileName = place.first;
+                            string suffix = "_io_l";
+
+                            if (fromModule)
+                                suffix = "_io_m";
+
+                            auto origCopy = copyArray(place, array_to_copy, linesByFile.second, suffix + to_string(region->GetId()), fileName, newDeclsToInclude, copied);
+                            SgStatement* decl = SgStatement::getStatementByFileAndLine(place.first, place.second);
+
+                            if(decl)
+                                decl = decl->lexNext();
+
+                            if(decl)
+                            {
+                                string dir_str;
+                                if (decl->comments())
+                                {
+                                    string str_comment = string(decl->comments());
+                                    if(str_comment.size() && str_comment.back() != '\n')
+                                        dir_str += "\n";
+                                }
+
+                                dir_str += "!$SPF ANALYSIS(PROCESS_PRIVATE(" + string(origCopy.second->identifier()) + "))\n";
+                                decl->addComment(dir_str.c_str());
+                            }
+                            created_copies.insert({ array_to_copy, origCopy.second });
+                        }
+                    }
+
+                    curr_stmt = curr_stmt->lexNext();
+                }
+            }
+        }
+    }
+}

sapfor/experts/Sapfor_2017/_src/Transformations/replace_dist_arrays_in_io.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include "../ParallelizationRegions/ParRegions.h"
+#include "../Utils/SgUtils.h"
+#include "../Utils/errors.h"
+#include "../GraphCall/graph_calls.h"
+
+void replaceDistributedArraysInIO(std::vector<ParallelRegion*>& regions, 
+                                  const std::map<std::string, std::vector<FuncInfo*>>& allFuncInfo, 
+                                  std::map<std::string, std::vector<Messages>>& SPF_messages,
+                                  std::map<std::string, std::map<int, std::set<std::string>>>& newDeclsToInclude);

sapfor/experts/Sapfor_2017/_src/Utils/PassManager.h

@@ -257,6 +257,8 @@ void InitPassesDependencies(map<passes, vector<passes>> &passDepsIn, set<passes>
 
     list({ REVERT_SUBST_EXPR_RD, CONVERT_LOOP_TO_ASSIGN }) <= Pass(RESOLVE_PAR_REGIONS);
 
+    list({ REVERT_SUBST_EXPR_RD, CONVERT_LOOP_TO_ASSIGN, FILL_PAR_REGIONS}) <= Pass(REMOVE_DIST_ARRAYS_FROM_IO);
+
     Pass(REVERT_SUBST_EXPR_RD) <= Pass(EXPAND_EXTRACT_PAR_REGION);
     
     Pass(FILL_PAR_REGIONS) <= Pass(PRINT_PAR_REGIONS_ERRORS);