#include "leak_detector.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../Distribution/Arrays.h" #include "graph_calls.h" #include "../GraphLoop/graph_loops.h" #include "../ParallelizationRegions/ParRegions.h" #include "remote_access.h" using std::vector; using std::pair; using std::tuple; using std::map; using std::set; using std::make_pair; using std::make_tuple; using std::get; using std::string; using std::wstring; #define DEB 0 static set fillRemotesInParallel(const ParallelDirective* dvm_dir) { set remotesInParallel; for (auto& elem : dvm_dir->remoteAccess) remotesInParallel.insert(elem.first.first.first + "(" + elem.first.second + ")"); return remotesInParallel; } static void createRemoteInParallel(const pair& under_dvm_dir, const set& doneInLoops, const map& funcMap, map& uniqRemotes, vector& messages, const DataDirective& data, const vector& currVar, const uint64_t regionId, const map>& arrayLinksByFuncCalls) { const set usedArrays = under_dvm_dir.first->usedArrays; const set usedArraysWrite = under_dvm_dir.first->usedArraysWrite; const set remotesInParallel = fillRemotesInParallel(under_dvm_dir.second); set addedRemotes; for (auto& usedArr : usedArrays) { if (doneInLoops.find(usedArr) == doneInLoops.end() && usedArraysWrite.find(usedArr) == usedArraysWrite.end()) { set realRefs; getRealArrayRefs(usedArr, usedArr, realRefs, arrayLinksByFuncCalls); bool isDistr = false; for (auto& realRef : realRefs) { auto templ = realRef->GetTemplateArray(regionId); for (int z = 0; z < data.distrRules.size(); ++z) { if (templ == data.distrRules[z].first) { for (auto& elem : data.distrRules[z].second[currVar[z]].distRule) if (elem == BLOCK) isDistr = true; break; } } } if (isDistr) { vector mapToLoop; for (int z = 0; z < usedArr->GetDimSize(); ++z) mapToLoop.push_back(""); //TODO: find all array refs addRemoteLink(under_dvm_dir.first, funcMap, createRemoteLink(under_dvm_dir.first, usedArr), uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, -1, false); __spf_print(DEB, "CRIP: %d, AFTER MAIN CHECK for arrray '%s'\n", __LINE__, usedArr->GetShortName().c_str()); } } } } static void createRemoteInParallel(const map> remoteRegularReadsOfTopLoop, const pair under_dvm_dir, const DIST::Arrays& allArrays, const map>& loopInfo, DIST::GraphCSR& reducedG, const DataDirective& data, const vector& currVar, map& uniqRemotes, vector& messages, const uint64_t regionId, const map>& arrayLinksByFuncCalls, set& doneInLoops, const map& funcMap) { if (!under_dvm_dir.first || !under_dvm_dir.second) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); __spf_print(DEB, "createRemoteInParallel %d, for loop %d\n", __LINE__, under_dvm_dir.first->lineNum); set addedRemotes; auto it = loopInfo.find(under_dvm_dir.first); if (it == loopInfo.end()) { if (under_dvm_dir.first->perfectLoop > 1) { pair nextDir = under_dvm_dir; nextDir.first = under_dvm_dir.first->children[0]; createRemoteInParallel(remoteRegularReadsOfTopLoop, nextDir, allArrays, loopInfo, reducedG, data, currVar, uniqRemotes, messages, regionId, arrayLinksByFuncCalls, doneInLoops, funcMap); } return; } const map& currInfo = it->second; const ParallelDirective* parDir = under_dvm_dir.second; DIST::Array* arrayRefOnDir = parDir->arrayRef; set realRefArrayOnDir; const set remotesInParallel = fillRemotesInParallel(parDir); const LoopGraph* currLoop = under_dvm_dir.first; if (!arrayRefOnDir->IsTemplate()) { getRealArrayRefs(arrayRefOnDir, arrayRefOnDir, realRefArrayOnDir, arrayLinksByFuncCalls); if (realRefArrayOnDir.size() != 1) { vector>>> allRules(realRefArrayOnDir.size()); int tmpIdx = 0; for (auto& array : realRefArrayOnDir) reducedG.GetAlignRuleWithTemplate(array, allArrays, allRules[tmpIdx++], regionId); if (!isAllRulesEqual(allRules)) { __spf_print(1, "not supported yet\n"); printInternalError(convertFileName(__FILE__).c_str(), __LINE__); } else arrayRefOnDir = *(realRefArrayOnDir.begin()); } else arrayRefOnDir = *(realRefArrayOnDir.begin()); } // for all array accesses in loop for (auto& array : currInfo) { DIST::Array* arrayRef = array.first; if (!arrayRef) continue; doneInLoops.insert(arrayRef); const ArrayInfo* currArrayInfo = array.second; set realArrayRef; getRealArrayRefs(arrayRef, arrayRef, realArrayRef, arrayLinksByFuncCalls); for (auto& elem : realArrayRef) { arrayRef = elem; // fill links between current array and array in parallel dir vector links; if (arrayRef != arrayRefOnDir) links = findLinksBetweenArrays(arrayRef, arrayRefOnDir, regionId); else { links.resize(arrayRef->GetDimSize()); for (int k = 0; k < arrayRef->GetDimSize(); ++k) links[k] = k; } //fill info links with template auto linksWithTempl = arrayRef->GetLinksWithTemplate(regionId); auto alignRuleWithTempl = arrayRef->GetAlignRulesWithTemplate(regionId); const DIST::Array* templArray = arrayRef->GetTemplateArray(regionId); if (!templArray) continue; // may be error? DIST::Array* templRefOnDir = arrayRefOnDir; if (!templRefOnDir->IsTemplate()) templRefOnDir = templRefOnDir->GetTemplateArray(regionId); if (templArray != templRefOnDir) // different templates { doneInLoops.erase(array.first); continue; } // fill distribute data variant const DistrVariant* distrVar = NULL; for (int k = 0; k < data.distrRules.size(); ++k) { if (data.distrRules[k].first == templArray) { distrVar = &data.distrRules[k].second[currVar[k]]; break; } } if (!distrVar) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); // set new redistribute rule, if exist const DistrVariant* newDistVar = currLoop->getRedistributeRule(templArray); if (newDistVar) distrVar = newDistVar; set parallelVars; parallelVars.insert(parDir->parallel.begin(), parDir->parallel.end()); vector mapToLoop; for (int i = 0; i < links.size(); ++i) { if (links[i] != -1 && linksWithTempl[i] != -1) { bool isCorrect = false; const bool isInParallel = parallelVars.find(parDir->on[links[i]].first) != parallelVars.end(); if (distrVar->distRule[linksWithTempl[i]] == BLOCK) { if (parDir->on[links[i]].first != "*" && isInParallel) isCorrect = true; else isCorrect = false; } if (isCorrect) mapToLoop.push_back(parDir->on[links[i]].first); else mapToLoop.push_back(""); } else mapToLoop.push_back(""); } // main check for (int i = 0; i < links.size(); ++i) { bool needToCheck = false; if (links[i] != -1 && linksWithTempl[i] != -1) { const bool isInParallel = parallelVars.find(parDir->on[links[i]].first) != parallelVars.end(); if (distrVar->distRule[linksWithTempl[i]] == BLOCK) { if (parDir->on[links[i]].first != "*" && !isInParallel) needToCheck = false; else needToCheck = true; } } else if (linksWithTempl[i] != -1) { // if distributed and used in loop with out link with array on dir if (distrVar->distRule[linksWithTempl[i]] == BLOCK) { // check all unrecognized refs for (auto& unrec : currArrayInfo->arrayAccessUnrec) { if (addedRemotes.find(unrec.first) != addedRemotes.end()) continue; else if (unrec.second.second[i] == REMOTE_TRUE) { addRemoteLink(currLoop, funcMap, unrec.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, unrec.second.first); __spf_print(DEB, "CRIP: %d, REMOTE_TRUE\n", __LINE__); } } // check all regular refs for (auto& reg : currArrayInfo->arrayAccess) { if (addedRemotes.find(reg.first) != addedRemotes.end()) continue; if (arrayRef == arrayRefOnDir) continue; addRemoteLink(currLoop, funcMap, reg.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, reg.second.first); __spf_print(DEB, "CRIP: %d, IRREG_REFS\n", __LINE__); } continue; } } // check if current dimention is distributed if (needToCheck) { // check unregular acceses for (auto& unrec : currArrayInfo->arrayAccessUnrec) { if (addedRemotes.find(unrec.first) != addedRemotes.end()) continue; if (unrec.second.second[i] == REMOTE_TRUE) { addRemoteLink(currLoop, funcMap, unrec.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, unrec.second.first); __spf_print(DEB, "CRIP: %d, IRREG_REFS && REMOTE_TRUE\n", __LINE__); } } // and check regular acceses for (auto& regAccess : currArrayInfo->arrayAccess) { if (addedRemotes.find(regAccess.first) != addedRemotes.end()) continue; if (arrayRef == arrayRefOnDir) continue; //if has reads in more than one dim and it dims are dirstributed int countOfDimAcc = 0; for (int z = 0; z < linksWithTempl.size(); ++z) { bool distributed = false; if (linksWithTempl[z] != -1) distributed = (distrVar->distRule[linksWithTempl[z]] == BLOCK); countOfDimAcc += ((regAccess.second.second[z].coefficients.size() != 0) && distributed) ? 1 : 0; } if (countOfDimAcc > 1 || countOfDimAcc == 1 && parDir->on[links[i]].first == "*") { addRemoteLink(currLoop, funcMap, regAccess.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, regAccess.second.first); __spf_print(DEB, "CRIP: %d ---\n", __LINE__); continue; } //if has distributed dim but loop maped to * if (parDir->on[links[i]].first == "*") { if (linksWithTempl[i] != -1) if (distrVar->distRule[linksWithTempl[i]] == BLOCK) { addRemoteLink(currLoop, funcMap, regAccess.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, regAccess.second.first); __spf_print(DEB, "CRIP: %d, NOT MAPPED\n", __LINE__); continue; } } //if this array has no map rules to current array and this dim is distributed if (currLoop->directiveForLoop) { if (currLoop->directiveForLoop->on[links[i]].first != "*") { if (regAccess.second.second[i].coefficients.size() == 0) { addRemoteLink(currLoop, funcMap, regAccess.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, regAccess.second.first); __spf_print(DEB, "CRIP: %d, ----\n", __LINE__); continue; } } } // check with loop alignment auto itRegRemote = remoteRegularReadsOfTopLoop.find(array.first); if (itRegRemote != remoteRegularReadsOfTopLoop.end()) { bool wasAdd = false; for (auto& reads : regAccess.second.second[i].coefficients) { const pair& currReadAcc = reads.first; for (auto& ref : itRegRemote->second[i].coefficients) { if (ref.first == currReadAcc) { addRemoteLink(currLoop, funcMap, regAccess.first, uniqRemotes, remotesInParallel, addedRemotes, mapToLoop, messages, regAccess.second.first); __spf_print(DEB, "CRIP: %d, MISS\n", __LINE__); wasAdd = true; break; } } if (wasAdd) break; } } } } } // main check } } if (currLoop->perfectLoop > 1) { pair nextDir = under_dvm_dir; nextDir.first = currLoop->children[0]; createRemoteInParallel(remoteRegularReadsOfTopLoop, nextDir, allArrays, loopInfo, reducedG, data, currVar, uniqRemotes, messages, regionId, arrayLinksByFuncCalls, doneInLoops, funcMap); } } map createRemoteInParallel(const pair under_dvm_dir, const DIST::Arrays& allArrays, const map>& loopInfo, DIST::GraphCSR& reducedG, const DataDirective& data, const vector& currVar, vector& messages, const uint64_t regionId, const map>& arrayLinksByFuncCalls, const map& funcMap) { map uniqRemotes; set doneInLoops; createRemoteInParallel(under_dvm_dir.first->remoteRegularReads, under_dvm_dir, allArrays, loopInfo, reducedG, data, currVar, uniqRemotes, messages, regionId, arrayLinksByFuncCalls, doneInLoops, funcMap); createRemoteInParallel(under_dvm_dir, doneInLoops, funcMap, uniqRemotes, messages, data, currVar, regionId, arrayLinksByFuncCalls); return uniqRemotes; } static void addInfoToMap(map>& loopInfo, LoopGraph* position, DIST::Array* array, ArrayRefExp* arrayRef, const int dimNum, const REMOTE_TYPE& value, const int currLine, const int maxDimSize, vector& requests) { requests.push_back(RemoteRequest(position, dimNum, value, currLine, maxDimSize)); auto it = loopInfo.find(position); if (loopInfo.end() == it) it = loopInfo.insert(it, make_pair(position, map())); auto it1 = it->second.find(array); if (it1 == it->second.end()) it1 = it->second.insert(it1, make_pair(array, new ArrayInfo())); auto it2 = it1->second->arrayAccessUnrec.find(arrayRef); if (it2 == it1->second->arrayAccessUnrec.end()) { it2 = it1->second->arrayAccessUnrec.insert(it2, make_pair(arrayRef, make_pair(currLine, vector()))); it2->second.second.resize(maxDimSize); std::fill(it2->second.second.begin(), it2->second.second.end(), REMOTE_NONE); } if (dimNum == -1) { for (int z = 0; z < it2->second.second.size(); ++z) it2->second.second[z] |= value; } else it2->second.second[dimNum] |= value; if (value == REMOTE_TRUE) __spf_print(DEB, "RemoteAccess[%d]: true for dim %d and array %s, loop line %d\n", __LINE__, dimNum, array->GetShortName().c_str(), position->lineNum); } static pair getDistrVariant(const vector>>& distrRules, DIST::Array* forTempl, const vector& currentVariant) { pair currentVar; for (int z1 = 0; z1 < currentVariant.size(); ++z1) { if (distrRules[z1].first == forTempl) { currentVar = make_pair(distrRules[z1].first, &distrRules[z1].second[currentVariant[z1]]); break; } } if (!currentVar.first) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); return currentVar; } // call this function for each array reference under loop(s) vector checkArrayRefInLoopForRemoteStatus(bool ifUnknownArrayAssignFound, int sumMatched, // sum of matched dimensions of 'arrayRef' to each 'parentLoops' int numOfSubs, // number of subscriptions of 'arrayRef' int maxMatched, // maximum numbe of matched dimensions of 'arrayRef' to each 'parentLoops' int currLine, // current line in source code of 'arrayRef' DIST::Array* currArray, // DIST::Array of 'arrayRef' vector& wasFoundForLoop, // size == 'parentLoops'.size(), init value -> 0, each position is sum of match dims count of 'arrayRef' ArrayRefExp* arrayRef, // current expression of array ref under all 'parentLoops', map>& loopInfo, const vector& matchedToDim, // size == 'parentLoops'.size(), init value -> -1, each position is dimension of match 'arrayRef' for 'parentLoop' const map& sortedLoopGraph, const map>& arrayLinksByFuncCalls, const ParallelRegion* region, const vector& parentLoops) // all loops which are associated with 'arrayRef' { vector requests; if (sumMatched != parentLoops.size() && sumMatched == numOfSubs) { set realArrayRefs; getRealArrayRefs(currArray, currArray, realArrayRefs, arrayLinksByFuncCalls); bool ok = true; DIST::Array* templ = NULL; vector alignCoefs; for (auto& real : realArrayRefs) { DIST::Array* curr = real->GetTemplateArray(region->GetId(), false); alignCoefs = real->GetLinksWithTemplate(region->GetId()); if (templ == NULL) templ = curr; else if (templ != curr) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); } string debOutStr = ""; char buf[256]; sprintf(buf, "RemoteAccess[%d]: check aligns for '%s'\n", __LINE__, currArray->GetShortName().c_str()); debOutStr += buf; for (int z = 0; z < wasFoundForLoop.size(); ++z) { sprintf(buf, "RemoteAccess[%d]: check aligns %d == %d\n", __LINE__, z, wasFoundForLoop[z]); debOutStr += buf; } for (int z = 0; z < matchedToDim.size(); ++z) { sprintf(buf, "RemoteAccess[%d]: matchedToDim[%d] = %d\n", __LINE__, z, matchedToDim[z]); debOutStr += buf; } for (int l = 0; l < alignCoefs.size(); ++l) { sprintf(buf, "RemoteAccess[%d]: alignCoefs[%d] = %d\n", __LINE__, l, alignCoefs[l]); debOutStr += buf; } //check array's alignment for (int z = 0; z < wasFoundForLoop.size() && ok; ++z) { auto it = sortedLoopGraph.find(parentLoops[z]->lineNum); if (it == sortedLoopGraph.end()) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); if (!templ) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); const DataDirective& dataDirectives = region->GetDataDir(); const vector& currentVariant = region->GetCurrentVariant(); pair currentVar = getDistrVariant(dataDirectives.distrRules, templ, currentVariant); LoopGraph* loop = it->second; if (!loop->directiveForLoop) { if (wasFoundForLoop[z]) { if (matchedToDim[z] != -1 && alignCoefs[matchedToDim[z]] != -1 && currentVar.second->distRule[alignCoefs[matchedToDim[z]]] == distType::BLOCK) { ok = false; if (!ok) { __spf_print(DEB, "%s\n", debOutStr.c_str()); __spf_print(DEB, "RemoteAccess[%d]: call addInfoMaps from aligns miss\n", __LINE__); __spf_print(DEB, "RemoteAccess[%d]: z = %d\n", __LINE__, z); addInfoToMap(loopInfo, parentLoops[z], currArray, arrayRef, matchedToDim[z], REMOTE_TRUE, currLine, numOfSubs, requests); } } } continue; } //apply redistribute if (loop->getRedistributeRule(currentVar.first) != NULL) currentVar.second = loop->getRedistributeRule(currentVar.first); DIST::Array* loopT = loop->directiveForLoop->arrayRef; sprintf(buf, "RemoteAccess[%d]: z = %d, array '%s'\n", __LINE__, z, loopT->GetShortName().c_str()); debOutStr += buf; int dimToMap = -1; for (int z1 = 0; z1 < loopT->GetDimSize(); ++z1) if (loop->directiveForLoop->on[z1].first != "*") dimToMap = z1; sprintf(buf, "RemoteAccess[%d]: z = %d, dimToMap = %d\n", __LINE__, z, dimToMap); debOutStr += buf; if (dimToMap != -1) { if (loopT != templ && !loopT->IsTemplate()) { sprintf(buf, "RemoteAccess[%d]: z = %d, false check !=\n", __LINE__, z); debOutStr += buf; DIST::Array* loopTempl = loopT->GetTemplateArray(region->GetId(), false); vector loopAlignCoefs = loopT->GetLinksWithTemplate(region->GetId()); if (loopTempl == NULL) { set tmpSet; getRealArrayRefs(loopT, loopT, tmpSet, arrayLinksByFuncCalls); set templates; for (auto& elem : tmpSet) { loopTempl = elem->GetTemplateArray(region->GetId(), false); loopAlignCoefs = elem->GetLinksWithTemplate(region->GetId()); templates.insert(loopTempl); } if (templates.size() != 1) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); } if (loopTempl) { sprintf(buf, "RemoteAccess[%d]: z = %d, array '%s'\n", __LINE__, z, loopTempl->GetShortName().c_str()); debOutStr += buf; } if (templ != loopTempl) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); if (loopAlignCoefs.size() == 0) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); if (loopAlignCoefs.size() <= dimToMap) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); if (loopAlignCoefs[dimToMap] == -1) { //TODO: need more tests for this case: prev bug => bugreport_1676461809 /*if (loop->hasParalleDirectiveBefore()) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); else*/ continue; } else dimToMap = loopAlignCoefs[dimToMap]; } else if (loopT != templ && loopT->IsTemplate()) { bool isDistr = false; for (auto& elem : currentVar.second->distRule) if (elem == BLOCK) isDistr = true; if (isDistr) { ok = false; __spf_print(DEB, "%s\n", debOutStr.c_str()); __spf_print(DEB, "RemoteAccess[%d]: call addInfoMaps from template miss\n", __LINE__); __spf_print(DEB, "RemoteAccess[%d]: z = %d\n", __LINE__, z); addInfoToMap(loopInfo, parentLoops[z], currArray, arrayRef, matchedToDim[z], REMOTE_TRUE, currLine, numOfSubs, requests); } continue; } sprintf(buf, "RemoteAccess[%d]: ** z = %d, dimToMap = %d\n", __LINE__, z, dimToMap); debOutStr += buf; for (int z = 0; z < currentVar.second->distRule.size(); ++z) { sprintf(buf, "RemoteAccess[%d]: distRule[%d] = %d\n", __LINE__, z, currentVar.second->distRule[z]); debOutStr += buf; } if (wasFoundForLoop[z]) { if (matchedToDim[z] != -1 && currentVar.second->distRule[alignCoefs[matchedToDim[z]]] == distType::BLOCK) { bool found = false; for (int l = 0; l < alignCoefs.size(); ++l) { if (alignCoefs[l] == dimToMap) found = true; } ok = found; if (!ok) { __spf_print(DEB, "%s\n", debOutStr.c_str()); __spf_print(DEB, "RemoteAccess[%d]: call addInfoMaps from aligns miss\n", __LINE__); __spf_print(DEB, "RemoteAccess[%d]: z = %d\n", __LINE__, z); __spf_print(DEB, "RemoteAccess[%d]: dimToMap = %d\n", __LINE__, dimToMap); addInfoToMap(loopInfo, parentLoops[z], currArray, arrayRef, matchedToDim[z], REMOTE_TRUE, currLine, numOfSubs, requests); } } } else { bool distrAny = false; bool distrAll = true; int dDim = -1; for (int idx = 0; idx < alignCoefs.size(); ++idx) { if (currentVar.second->distRule[idx] == distType::BLOCK) { dDim = idx; distrAny = true; } else distrAll = false; } if (matchedToDim[z] == -1 && distrAny && !distrAll) { ok = false; if (!ok) { __spf_print(DEB, "%s\n", debOutStr.c_str()); __spf_print(DEB, "RemoteAccess[%d]: call addInfoMaps from aligns miss\n", __LINE__); __spf_print(DEB, "RemoteAccess[%d]: z = %d\n", __LINE__, z); __spf_print(DEB, "RemoteAccess[%d]: dimToMap = %d\n", __LINE__, dimToMap); addInfoToMap(loopInfo, parentLoops[z], currArray, arrayRef, dDim, REMOTE_TRUE, currLine, numOfSubs, requests); } } } } } if (ok && currArray->GetDimSize() > 1) for (int z = 0; z < wasFoundForLoop.size(); ++z) wasFoundForLoop[z] = 1; } if (ifUnknownArrayAssignFound) { if (sumMatched != numOfSubs || maxMatched != 1 || (sumMatched != parentLoops.size() && sumMatched != numOfSubs)) { int local = 0; bool hasLimits = false; for (int i = 0; i < wasFoundForLoop.size(); ++i) { if (wasFoundForLoop[i] == 1) { auto itLoop = sortedLoopGraph.find(parentLoops[i]->lineNum); if (itLoop == sortedLoopGraph.end()) printInternalError(convertFileName(__FILE__).c_str(), __LINE__); if (itLoop->second->hasLimitsToParallel()) hasLimits = true; } } for (int i = 0; i < wasFoundForLoop.size(); ++i) { if (wasFoundForLoop[i] != 1) { for (int k = 0; k < numOfSubs; ++k) { if (hasLimits) { __spf_print(DEB, "RemoteAccess[%d]: call addInfoMaps from hasLimits\n", __LINE__); addInfoToMap(loopInfo, parentLoops[i], currArray, arrayRef, k, REMOTE_TRUE, currLine, numOfSubs, requests); } } } } } } return requests; }