shared memory: private arrays, free loops

2024-03-26 13:59:48 +03:00
parent 83f4e1289c
commit ebca0b4c41
3 changed files with 229 additions and 37 deletions
--- a/sapfor/experts/Sapfor_2017/_src/LoopAnalyzer/loop_analyzer.cpp
+++ b/sapfor/experts/Sapfor_2017/_src/LoopAnalyzer/loop_analyzer.cpp
@@ -125,8 +125,7 @@ static void addInfoToMap(map<SgForStmt*, map<SgSymbol*, ArrayInfo>> &loopInfo, S
        __spf_print(DEB, "RemoteAccess[%d]: true for dim %d and array %s, loop line %d\n", __LINE__, dimNum, symb->identifier(), position->lineNumber());
 }

-enum { READ_OP, WRITE_OP, UNREC_OP };
-static void addInfoToVectors(map<SgForStmt*, map<SgSymbol*, ArrayInfo>> &loopInfo, SgForStmt *position, SgSymbol *symb, 
+void addInfoToVectors(map<SgForStmt*, map<SgSymbol*, ArrayInfo>> &loopInfo, SgForStmt *position, SgSymbol *symb, 
                      const int dimNum, const pair<int, int> newCoef, int type, const int maxDimSize, const double currentW)
 {
    auto itLoop = loopInfo.find(position);
@@ -159,7 +158,7 @@ static void addInfoToVectors(map<SgForStmt*, map<SgSymbol*, ArrayInfo>> &loopInf
 }


-static vector<int> matchSubscriptToLoopSymbols(const vector<SgForStmt*> &parentLoops, SgExpression *subscr,
+vector<int> matchSubscriptToLoopSymbols(const vector<SgForStmt*> &parentLoops, SgExpression *subscr,
                                        SgArrayRefExp *arrayRefIn, const int side, const int dimNum,
                                        map<SgForStmt*, map<SgSymbol*, ArrayInfo>> &loopInfo,
                                        const int currLine, const int numOfSubscriptions, const double currentW)
@@ -557,7 +556,7 @@ static void mapArrayRef(SgStatement* currentSt, SgExpression* currExp,
    __spf_print(PRINT_ARRAY_ARCS, "\n");
 }

-void findArrayRef(const vector<SgForStmt*> &parentLoops, SgExpression *currExp, const int lineNum, const int side, 
+static void findArrayRef(const vector<SgForStmt*> &parentLoops, SgExpression *currExp, const int lineNum, const int side, 
                         map<SgForStmt*, map<SgSymbol*, ArrayInfo>> &loopInfo, const set<string> &privatesVars, 
                         vector<set<string>>& privatesVarsForLoop, map<int, LoopGraph*> &sortedLoopGraph, 
                         const map<string, vector<SgExpression*>> &commonBlocks,
--- a/sapfor/experts/Sapfor_2017/_src/LoopAnalyzer/loop_analyzer_internal.h
+++ b/sapfor/experts/Sapfor_2017/_src/LoopAnalyzer/loop_analyzer_internal.h
@@ -58,13 +58,13 @@ void changeLoopWeight(double& currentWeight, const std::map<int, LoopGraph*>& so

 SgStatement* takeOutConditions(std::stack<SgExpression*>& conditions, std::stack<SgStatement*>& ifBlocks, SgStatement* st);

-void findArrayRef(const std::vector<SgForStmt*>& parentLoops, SgExpression* currExp, const int lineNum, const int side,
-                  std::map<SgForStmt*, std::map<SgSymbol*, ArrayInfo>>& loopInfo, const std::set<std::string>& privatesVars,
-                  std::vector<std::set<std::string>>& privatesVarsForLoop, std::map<int, LoopGraph*>& sortedLoopGraph,
-                  const std::map<std::string, std::vector<SgExpression*>>& commonBlocks,
-                  const std::map<std::tuple<int, std::string, std::string>, std::pair<DIST::Array*, DIST::ArrayAccessInfo*>>& declaredArrays,
-                  bool wasDistributedArrayRef, std::map<std::string, std::pair<SgSymbol*, SgStatement*>>& notMappedDistributedArrays,
-                  std::set<std::string>& mappedDistrbutedArrays, SgStatement* currentSt, const ParallelRegion* reg, const double currentW,
-                  const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls);
+enum { READ_OP, WRITE_OP, UNREC_OP };
+void addInfoToVectors(std::map<SgForStmt*, std::map<SgSymbol*, ArrayInfo>>& loopInfo, SgForStmt* position, SgSymbol* symb,
+                      const int dimNum, const std::pair<int, int> newCoef, int type, const int maxDimSize, const double currentW);
+
+std::vector<int> matchSubscriptToLoopSymbols(const std::vector<SgForStmt*>& parentLoops, SgExpression* subscr,
+                                             SgArrayRefExp* arrayRefIn, const int side, const int dimNum,
+                                             std::map<SgForStmt*, std::map<SgSymbol*, ArrayInfo>>& loopInfo,
+                                             const int currLine, const int numOfSubscriptions, const double currentW);

 bool hasNonPureFunctions(SgExpression* ex, LoopGraph* loopRef, std::vector<Messages>& messagesForFile, const int line, const std::map<std::string, FuncInfo*>& funcByName);
--- a/sapfor/experts/Sapfor_2017/_src/LoopAnalyzer/loop_analyzer_nodist.cpp
+++ b/sapfor/experts/Sapfor_2017/_src/LoopAnalyzer/loop_analyzer_nodist.cpp
@@ -36,7 +36,6 @@ extern void createMapLoopGraph(map<int, LoopGraph*>& sortedLoopGraph, const vect
 extern map<DIST::Array*, std::tuple<int, string, string>> tableOfUniqNamesByArray;
 static void convertOneLoopNoDist(LoopGraph* currLoop, map<LoopGraph*, map<DIST::Array*, ArrayInfo*>>& outInfo,
                                 const map<SgSymbol*, ArrayInfo>& toConvert,
-                                 const set<string>& privateArrays,
                                 const map<string, vector<SgExpression*>>& commonBlocks,
                                 const map<tuple<int, string, string>, pair<DIST::Array*, DIST::ArrayAccessInfo*>>& declaredArrays,
                                 const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls,
@@ -109,7 +108,6 @@ static void convertOneLoopNoDist(LoopGraph* currLoop, map<LoopGraph*, map<DIST::
 static map<LoopGraph*, map<DIST::Array*, ArrayInfo*>>
 convertLoopInfoNoDist(const map<SgForStmt*, map<SgSymbol*, ArrayInfo>>& loopInfo,
                      const map<int, LoopGraph*>& sortedLoopGraph,
-                      const set<string>& privateArrays,
                      const map<string, vector<SgExpression*>>& commonBlocks,
                      const map<tuple<int, string, string>, pair<DIST::Array*, DIST::ArrayAccessInfo*>>& declaredArrays,
                      const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls,
@@ -123,12 +121,188 @@ convertLoopInfoNoDist(const map<SgForStmt*, map<SgSymbol*, ArrayInfo>>& loopInfo
        if (itGraph == sortedLoopGraph.end())
            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);

-        convertOneLoopNoDist(itGraph->second, outInfo, it->second, privateArrays, commonBlocks, declaredArrays, arrayLinksByFuncCalls, createdArrays);
+        convertOneLoopNoDist(itGraph->second, outInfo, it->second, commonBlocks, declaredArrays, arrayLinksByFuncCalls, createdArrays);
    }

    return outInfo;
 }

+static vector<int> matchArrayToLoopSymbols(const vector<SgForStmt*> &parentLoops, vector<set<string>>& privatesVarsForLoop,
+                                           SgExpression *currExp, const int side,
+                                           map<SgForStmt*, map<SgSymbol*, ArrayInfo>> &loopInfo, const int currLine,
+                                           map<int, LoopGraph*> &sortedLoopGraph, const ParallelRegion *reg, const double currentW,
+                                           const map<DIST::Array*, set<DIST::Array*>> &arrayLinksByFuncCalls)
+{
+    SgArrayRefExp *arrayRef = (SgArrayRefExp*)currExp;
+    int numOfSubs = arrayRef->numberOfSubscripts();
+
+    currExp = currExp->lhs();    
+    vector<int> wasFoundForLoop(parentLoops.size());
+    vector<int> matched(numOfSubs);
+    vector<int> matchedToDim(parentLoops.size());
+    std::fill(wasFoundForLoop.begin(), wasFoundForLoop.end(), 0);
+    std::fill(matched.begin(), matched.end(), -1);
+    std::fill(matchedToDim.begin(), matchedToDim.end(), -1);
+    int maxMatched = 0;
+    int sumMatched = 0;
+
+    for (int i = 0; i < numOfSubs; ++i)
+    {
+        vector<int> matchToLoops = matchSubscriptToLoopSymbols(parentLoops, currExp->lhs(), arrayRef, side, i, loopInfo, currLine, numOfSubs, currentW);
+        for (int k = 0; k < matchToLoops.size(); ++k)
+        {
+            wasFoundForLoop[matchToLoops[k]]++;
+            matchedToDim[matchToLoops[k]] = i;
+        }
+
+        matched[i] = matchToLoops.size();
+        sumMatched += matchToLoops.size();
+        maxMatched = std::max(maxMatched, (int)matchToLoops.size());
+        currExp = currExp->rhs();
+    }
+
+    //full array is used, add unknown operations to all loops
+    if (numOfSubs == 0)
+    {
+        SgSymbol *currOrigArrayS = OriginalSymbol(arrayRef->symbol());
+        auto arrType = isSgArrayType(currOrigArrayS->type());
+        if (arrType != NULL)
+        {
+            for (int d = 0; d < arrType->dimension(); ++d)
+                for (int i = 0; i < parentLoops.size(); ++i)
+                    addInfoToVectors(loopInfo, parentLoops[i], currOrigArrayS, d, make_pair(0, 0), UNREC_OP, arrType->dimension(), currentW);
+        }
+    }
+
+    bool ifUnknownArrayAssignFound = false;
+    vector<int> canNotMapToLoop;
+    for (int i = 0; i < wasFoundForLoop.size(); ++i)
+    {
+        if (wasFoundForLoop[i] != 1 && 
+            privatesVarsForLoop[i].find(string(arrayRef->symbol()->identifier())) == privatesVarsForLoop[i].end())
+        {
+            auto itLoop = sortedLoopGraph.find(parentLoops[i]->lineNumber());
+            if (itLoop == sortedLoopGraph.end())
+                printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+            ifUnknownArrayAssignFound = true;
+            if (side == LEFT)
+                itLoop->second->hasUnknownArrayAssigns = true;
+
+            itLoop->second->hasUnknownDistributedMap = true;
+            canNotMapToLoop.push_back(parentLoops[i]->lineNumber());
+        }
+    }
+
+    if (side == LEFT)
+    {
+        if (ifUnknownArrayAssignFound)
+        {
+            const string arrayRefS = arrayRef->unparse();
+            for (auto &line : canNotMapToLoop)
+            {
+                __spf_print(1, "WARN: can not map write to array '%s' to loop on line %d\n", arrayRefS.c_str(), line);
+                wstring messageE, messageR;
+                __spf_printToLongBuf(messageE, L"can not map write to array '%s' to this loop", to_wstring(arrayRefS).c_str());
+
+                __spf_printToLongBuf(messageR, R59, to_wstring(arrayRefS).c_str());
+
+                if (line > 0)
+                    currMessages->push_back(Messages(WARR, line, messageR, messageE, 1025));
+            }
+        }
+    }
+
+    return wasFoundForLoop;
+}
+
+static void mapArrayRef(SgStatement* currentSt, SgExpression* currExp,
+                        const vector<SgForStmt*>& parentLoops, const int side, const int lineNum,
+                        map<SgForStmt*, map<SgSymbol*, ArrayInfo>>& loopInfo,
+                        vector<set<string>>& privatesVarsForLoop,
+                        map<int, LoopGraph*>& sortedLoopGraph, map<string, pair<SgSymbol*, SgStatement*>>& notMappedDistributedArrays,
+                        set<string>& mappedDistrbutedArrays,
+                        const ParallelRegion* reg, const double currentW, const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls)
+{
+    const char* printSide = NULL;
+    if (PRINT_ARRAY_ARCS)
+        printBlanks(2, (int)parentLoops.size());
+    if (side == LEFT)
+        printSide = "W_OP";
+    else
+        printSide = "R_OP";
+
+    __spf_print(PRINT_ARRAY_ARCS, "%s to array <%s> on line %d: ", printSide, OriginalSymbol(currExp->symbol())->identifier(), lineNum);
+    bool wasMapped = false;
+    vector<int> matched = matchArrayToLoopSymbols(parentLoops, privatesVarsForLoop, currExp, side, loopInfo, lineNum, sortedLoopGraph, reg, currentW, arrayLinksByFuncCalls);
+    for (int z = 0; z < matched.size(); ++z)
+        wasMapped |= (matched[z] != 0);
+
+    if (parentLoops.size() == 0)
+    {
+        SgSymbol* symb = currExp->symbol();
+        if (symb->type()->variant() == T_ARRAY)
+            notMappedDistributedArrays[symb->identifier()] = make_pair(symb, currentSt);
+    }
+    else
+    {
+        if (wasMapped)
+            mappedDistrbutedArrays.insert(currExp->symbol()->identifier());
+        else
+        {
+            SgSymbol* symb = currExp->symbol();
+            if (symb->type()->variant() == T_ARRAY)
+                notMappedDistributedArrays[symb->identifier()] = make_pair(symb, currentSt);
+        }
+    }
+    __spf_print(PRINT_ARRAY_ARCS, "\n");
+}
+
+static void findArrayRef(const vector<SgForStmt*>& parentLoops, SgExpression* currExp, const int lineNum, const int side,
+                         map<SgForStmt*, map<SgSymbol*, ArrayInfo>>& loopInfo,
+                         vector<set<string>>& privatesVarsForLoop, map<int, LoopGraph*>& sortedLoopGraph,
+                         map<string, pair<SgSymbol*, SgStatement*>>& notMappedDistributedArrays,
+                         set<string>& mappedDistrbutedArrays, SgStatement* currentSt, const ParallelRegion* reg, const double currentW,
+                         const map<DIST::Array*, set<DIST::Array*>>& arrayLinksByFuncCalls)
+{
+    int nextSide = side;
+    if (isArrayRef(currExp))
+    {
+        mapArrayRef(currentSt, currExp, parentLoops, side, lineNum, loopInfo, privatesVarsForLoop, sortedLoopGraph,
+                    notMappedDistributedArrays, mappedDistrbutedArrays, reg, currentW, arrayLinksByFuncCalls);
+        nextSide = (side == LEFT) ? RIGHT : side;
+    }
+
+    bool needToContinue = true;
+    if (currExp->variant() == FUNC_CALL)
+    {
+        SgFunctionCallExp* funcExp = (SgFunctionCallExp*)currExp;
+        auto currFunc = isUserFunctionInProject(funcExp->funName()->identifier());
+        if (currFunc)
+        {
+            for (int z = 0; z < funcExp->numberOfArgs(); ++z)
+            {
+                if ((currFunc->funcParams.inout_types[z] & OUT_BIT) != 0)
+                    nextSide = LEFT;
+                else
+                    nextSide = RIGHT;
+                findArrayRef(parentLoops, funcExp->arg(z), lineNum, nextSide, loopInfo, privatesVarsForLoop, sortedLoopGraph,
+                             notMappedDistributedArrays, mappedDistrbutedArrays, currentSt, reg, currentW, arrayLinksByFuncCalls);
+            }
+            needToContinue = false;
+        }
+    }
+
+    if (needToContinue)
+    {
+        if (currExp->lhs())
+            findArrayRef(parentLoops, currExp->lhs(), lineNum, nextSide, loopInfo, privatesVarsForLoop, sortedLoopGraph,
+                         notMappedDistributedArrays, mappedDistrbutedArrays, currentSt, reg, currentW, arrayLinksByFuncCalls);
+        if (currExp->rhs())
+            findArrayRef(parentLoops, currExp->rhs(), lineNum, nextSide, loopInfo, privatesVarsForLoop, sortedLoopGraph,
+                         notMappedDistributedArrays, mappedDistrbutedArrays, currentSt, reg, currentW, arrayLinksByFuncCalls);
+    }
+}
+
 void loopAnalyzerNoDist(SgFile* file, vector<ParallelRegion*>& regions, map<tuple<int, string, string>, DIST::Array*>& createdArrays,
                        vector<Messages>& messagesForFile, const map<string, vector<FuncInfo*>>& AllfuncInfo,
                        const map<tuple<int, string, string>, pair<DIST::Array*, DIST::ArrayAccessInfo*>>& declaredArrays,
@@ -187,8 +361,6 @@ void loopAnalyzerNoDist(SgFile* file, vector<ParallelRegion*>& regions, map<tupl
        map<SgForStmt*, map<SgSymbol*, ArrayInfo>> loopInfo;
        set<int> loopWithOutArrays;

-        set<string> privatesVars;
-
        SgStatement* st = file->functions(i);
        string funcName = "";
        if (st->variant() == PROG_HEDR)
@@ -210,6 +382,9 @@ void loopAnalyzerNoDist(SgFile* file, vector<ParallelRegion*>& regions, map<tupl
            funcName = funcH->symbol()->identifier();
        }

+        if (funcName == string("interp"))
+            int a = 0;
+
        vector<LoopGraph*> loopsForFunction;
        for (auto& loop : *loopGraph)
        {
@@ -269,7 +444,7 @@ void loopAnalyzerNoDist(SgFile* file, vector<ParallelRegion*>& regions, map<tupl
            const int currV = st->variant();
            if (currV == FOR_NODE)
            {   
-                tryToFindPrivateInAttributes(st, privatesVars);
+                //tryToFindPrivateInAttributes(st, privatesVars);

                set<string> toAdd;
                tryToFindPrivateInAttributes(st, toAdd);
@@ -312,12 +487,12 @@ void loopAnalyzerNoDist(SgFile* file, vector<ParallelRegion*>& regions, map<tupl
            else if (currV == ASSIGN_STAT)
            {
                if (st->expr(0))
-                    findArrayRef(parentLoops, st->expr(0), st->lineNumber(), LEFT, loopInfo, privatesVars, privatesVarsForLoop,
-                        sortedLoopGraph, commonBlocks, declaredArrays, false, notMappedDistributedArrays,
+                    findArrayRef(parentLoops, st->expr(0), st->lineNumber(), LEFT, loopInfo, privatesVarsForLoop,
+                        sortedLoopGraph, notMappedDistributedArrays,
                        mappedDistrbutedArrays, st, currReg, currentWeight, arrayLinksByFuncCalls);
                if (st->expr(1))
-                    findArrayRef(parentLoops, st->expr(1), st->lineNumber(), RIGHT, loopInfo, privatesVars, privatesVarsForLoop,
-                        sortedLoopGraph, commonBlocks, declaredArrays, false, notMappedDistributedArrays,
+                    findArrayRef(parentLoops, st->expr(1), st->lineNumber(), RIGHT, loopInfo, privatesVarsForLoop,
+                        sortedLoopGraph, notMappedDistributedArrays,
                        mappedDistrbutedArrays, st, currReg, currentWeight, arrayLinksByFuncCalls);
            }
            else if (currV == IF_NODE || currV == ELSEIF_NODE || currV == LOGIF_NODE || currV == SWITCH_NODE)
@@ -325,8 +500,8 @@ void loopAnalyzerNoDist(SgFile* file, vector<ParallelRegion*>& regions, map<tupl
                SgStatement* before = NULL;
                if (st->expr(0))
                {
-                    findArrayRef(parentLoops, st->expr(0), st->lineNumber(), RIGHT, loopInfo, privatesVars, privatesVarsForLoop,
-                        sortedLoopGraph, commonBlocks, declaredArrays, false, notMappedDistributedArrays,
+                    findArrayRef(parentLoops, st->expr(0), st->lineNumber(), RIGHT, loopInfo, privatesVarsForLoop,
+                        sortedLoopGraph, notMappedDistributedArrays,
                        mappedDistrbutedArrays, st, currReg, currentWeight, arrayLinksByFuncCalls);
                }
            }
@@ -344,12 +519,12 @@ void loopAnalyzerNoDist(SgFile* file, vector<ParallelRegion*>& regions, map<tupl
                        {
                            SgExpression* par = list->elem(z);
                            if ((func->funcParams.inout_types[z] & OUT_BIT) != 0)
-                                findArrayRef(parentLoops, par, st->lineNumber(), LEFT, loopInfo, privatesVars, privatesVarsForLoop,
-                                    sortedLoopGraph, commonBlocks, declaredArrays, false, notMappedDistributedArrays,
+                                findArrayRef(parentLoops, par, st->lineNumber(), LEFT, loopInfo, privatesVarsForLoop,
+                                    sortedLoopGraph, notMappedDistributedArrays,
                                    mappedDistrbutedArrays, st, currReg, currentWeight, arrayLinksByFuncCalls);
                            else
-                                findArrayRef(parentLoops, par, st->lineNumber(), RIGHT, loopInfo, privatesVars, privatesVarsForLoop,
-                                    sortedLoopGraph, commonBlocks, declaredArrays, false, notMappedDistributedArrays,
+                                findArrayRef(parentLoops, par, st->lineNumber(), RIGHT, loopInfo, privatesVarsForLoop,
+                                    sortedLoopGraph, notMappedDistributedArrays,
                                    mappedDistrbutedArrays, st, currReg, currentWeight, arrayLinksByFuncCalls);
                        }

@@ -385,8 +560,8 @@ void loopAnalyzerNoDist(SgFile* file, vector<ParallelRegion*>& regions, map<tupl

                    for (int z = 0; z < 3; ++z)
                        if (st->expr(z))
-                            findArrayRef(parentLoops, st->expr(z), st->lineNumber(), side, loopInfo, privatesVars, privatesVarsForLoop,
-                                sortedLoopGraph, commonBlocks, declaredArrays, false, notMappedDistributedArrays,
+                            findArrayRef(parentLoops, st->expr(z), st->lineNumber(), side, loopInfo, privatesVarsForLoop,
+                                sortedLoopGraph, notMappedDistributedArrays,
                                mappedDistrbutedArrays, st, currReg, currentWeight, arrayLinksByFuncCalls);

                }
@@ -395,7 +570,7 @@ void loopAnalyzerNoDist(SgFile* file, vector<ParallelRegion*>& regions, map<tupl
            st = st->lexNext();
        }

-        auto convertedLoopInfo = convertLoopInfoNoDist(loopInfo, sortedLoopGraph, privatesVars, commonBlocks, declaredArrays, arrayLinksByFuncCalls, createdArrays);
+        auto convertedLoopInfo = convertLoopInfoNoDist(loopInfo, sortedLoopGraph, commonBlocks, declaredArrays, arrayLinksByFuncCalls, createdArrays);

        processLoopInformationForFunction(convertedLoopInfo);

@@ -469,8 +644,26 @@ void loopAnalyzerNoDist(SgFile* file, vector<ParallelRegion*>& regions, map<tupl
        sendMessage_2lvl(L"");

        createParallelDirectivesNoDist(convertedLoopInfo, regions, arrayLinksByFuncCalls, messagesForFile);
-        if (parallizeFreeLoops)
-            selectFreeLoopsForParallelization(loopsForFunction, funcName, false, regions, messagesForFile);
+        for (auto& loopLine : loopWithOutArrays)
+        {
+            auto loopRef = sortedLoopGraph[loopLine];
+            if (loopRef->withoutDistributedArrays && loopRef->region && !loopRef->hasLimitsToParallel() && loopRef->lineNum > 0)
+            {
+                int nesting = 0;
+                LoopGraph* it = loopRef;
+                for (int z = 0; z < loopRef->perfectLoop; ++z, it->children.size() ? it = it->children[0] : it)
+                    if (it->withoutDistributedArrays && it->region && !it->hasLimitsToParallel() && it->lineNum > 0)
+                        ++nesting;
+
+                map<LoopGraph*, map<DIST::Array*, ArrayInfo*>> convertedLoopInfo;
+
+                it = loopRef;
+                for (int z = 0; z < nesting; ++z, it->children.size() ? it = it->children[0] : it)
+                    convertedLoopInfo.insert(make_pair(it, map<DIST::Array*, ArrayInfo*>()));
+
+                createParallelDirectivesNoDist(convertedLoopInfo, regions, map<DIST::Array*, set<DIST::Array*>>(), messagesForFile);
+            }
+        }

        __spf_print(PRINT_PROF_INFO, "Function ended\n");
    }