Alexander_KS/SAPFOR
4
2
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

merged master

Browse Source
This commit is contained in:
ALEXks
2025-05-27 19:00:57 +03:00
parent 00f205457c 8f55c13a0c
commit fca6b15cba
9 changed files with 188 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
src/Predictor/PredictScheme.h
View File

@@ -2,6 +2,7 @@
#include <vector>
#include "dvm.h"
#include "../GraphCall/graph_calls.h"
#include "../Utils/json.hpp"
class ParallelStats
{
@@ -11,10 +12,10 @@ public:
RemoteCount = ShadowCount = ReductionCount = AcrossCount = 0;
}
int RemoteCount;
int ShadowCount;
int ReductionCount;
int AcrossCount;
int RemoteCount = 0;
int ShadowCount = 0;
int ReductionCount = 0;
int AcrossCount = 0;
};
class PredictorStats
@@ -27,28 +28,32 @@ public:
}
ParallelStats ParallelStat;
int ParallelCount;
int RemoteCount;
int RedistributeCount;
int IntervalCount;
int TotalScoreComm;
int TotalScorePar;
int TotalScoreDist;
int ParallelCount = 0;
int RemoteCount = 0;
int RedistributeCount = 0;
int IntervalCount = 0;
int TotalScoreComm = 0;
int TotalScorePar = 0;
int TotalScoreDist = 0;
std::string to_string()
nlohmann::json toJson()
{
std::string res = "";
res += std::to_string(ParallelCount) + "|";
res += std::to_string(RemoteCount) + "|";
res += std::to_string(RedistributeCount) + "|";
res += std::to_string(IntervalCount) + "|";
nlohmann::json stat;
res += std::to_string(ParallelStat.RemoteCount) + "|";
res += std::to_string(ParallelStat.ShadowCount) + "|";
res += std::to_string(ParallelStat.ReductionCount) + "|";
res += std::to_string(ParallelStat.AcrossCount);
stat["ParallelCount"] = ParallelCount;
stat["RemoteCount"] = RemoteCount;
stat["RedistributeCount"] = RedistributeCount;
stat["IntervalCount"] = IntervalCount;
return res;
stat["PS_RemoteCount"] = ParallelStat.RemoteCount;
stat["PS_ShadowCount"] = ParallelStat.ShadowCount;
stat["PS_ReductionCount"] = ParallelStat.ReductionCount;
stat["PS_AcrossCount"] = ParallelStat.AcrossCount;
//TODO: need to improve
// (summed.TotalScoreComm != 0 ? 1.0 / summed.TotalScoreComm : 0.0) + (double)summed.TotalScorePar * 1000 + (countOfDist == 0 ? -5000 : countOfDist);
stat["TotalScore"] = -1 * (ParallelStat.RemoteCount + ParallelStat.ShadowCount + ParallelStat.ReductionCount + ParallelStat.AcrossCount);
return stat;
}
};

55
src/ProjectManipulation/ParseFiles.cpp
View File

@@ -387,7 +387,50 @@ static string shiftLines(const string &in, const map<string, const FileInfo*> &m
return newStr;
}
static int dumpErrors(const vector<FileInfo>& listOfProject, const vector<string>& errors)
static void addMessage(const string& in, const map<string, const FileInfo*>& mapOfFiles,
const FileInfo* currF, map<string, vector<Messages>>& messages, typeMessage type)
{
int byNum = 0;
auto it = in.find("on line ");
if (it != string::npos)
it += strlen("on line ");
int line = 0;
sscanf(in.c_str() + it, "%d", &line);
auto it1 = in.find("of", it + 1);
if (it1 == string::npos)
return;
it1 += 3;
string fileN = in.substr(it1, in.find(':', it1) - it1);
auto itF = mapOfFiles.find(fileN);
if (itF != mapOfFiles.end() && itF->second != currF)
{
byNum = itF->second->includesAdded;
if (byNum != 0)
{
if (line - byNum <= 0)
{
//return in;
printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
}
else
line -= byNum;
}
}
const string newStr = in.substr(0, it) + std::to_string(line) + in.substr(in.find(' ', it + 1));
wstring messageE, messageR;
__spf_printToLongBuf(messageE, L"%s", to_wstring(newStr).c_str());
__spf_printToLongBuf(messageR, L"%s", to_wstring(newStr).c_str());
messages[fileN].push_back(Messages(type, line, messageR, messageE, 6000));
}
static int dumpErrors(const vector<FileInfo>& listOfProject, const vector<string>& errors, map<string, vector<Messages>>& messages)
{
int errorsCount = 0;
map<string, const FileInfo*> mapOfFiles;
@@ -420,9 +463,13 @@ static int dumpErrors(const vector<FileInfo>& listOfProject, const vector<string
for (auto& elem : splited)
{
if (elem.find("Warning 308") != string::npos)
{
addMessage(elem, mapOfFiles, &file, messages, WARR);
outS += shiftLines(elem, mapOfFiles, &file) + "\n";
}
else if (elem.find("Error") != string::npos)
{
addMessage(elem, mapOfFiles, &file, messages, ERROR);
errS += shiftLines(elem, mapOfFiles, &file) + "\n";
errorsCount++;
}
@@ -652,7 +699,7 @@ static int parseFiles(vector<string>& errors, vector<FileInfo>& listOfProject, v
return rethrow;
}
int parseFiles(const char* proj, vector<string>& filesCompilationOrder, int parseForInlining)
int parseFiles(const char* proj, vector<string>& filesCompilationOrder, int parseForInlining, map<string, vector<Messages>>& messages)
{
FILE* list = fopen(proj, "r");
if (!list)
@@ -723,7 +770,7 @@ int parseFiles(const char* proj, vector<string>& filesCompilationOrder, int pars
vector<string> errors;
int rethrow = parseFiles(errors, listOfProject, filesCompilationOrder, parseForInlining);
int errCount = dumpErrors(listOfProject, errors);
int errCount = dumpErrors(listOfProject, errors, messages);
if (rethrow != 0)
throw rethrow;
@@ -732,6 +779,8 @@ int parseFiles(const char* proj, vector<string>& filesCompilationOrder, int pars
void parseFiles(int argc, char** argv)
{
map<string, vector<Messages>> messages;
bool isInline = false;
auto result = splitCommandLineForParse(argv, argc, isInline);
if (result.second.size() == 0)

2
src/ProjectManipulation/ParseFiles.h
View File

@@ -3,5 +3,5 @@
#include <vector>
#include <string>
int parseFiles(const char* proj, std::vector<std::string>& filesCompilationOrder, int parseForInlining);
int parseFiles(const char* proj, std::vector<std::string>& filesCompilationOrder, int parseForInlining, std::map<std::string, std::vector<Messages>>& messages);
void parseFiles(int argc, char** argv);

39
src/Sapfor.cpp
View File

@@ -64,6 +64,7 @@
#include "DvmhRegions/LoopChecker.h"
#include "DvmhRegions/ReadWriteAnalyzer.h"
#include "Utils/utils.h"
#include "Utils/json.hpp"
#include "Distribution/Array.h"
#include "VisualizerCalls/get_information.h"
@@ -109,6 +110,7 @@ using namespace std;
using std::chrono::high_resolution_clock;
using std::chrono::duration_cast;
using std::chrono::milliseconds;
using json = nlohmann::json;
int PASSES_DONE[EMPTY_PASS];
bool PASSES_DONE_INIT = false;
@@ -2053,7 +2055,7 @@ static void findFunctionsToInclude(bool needToAddErrors)
SPF_messages[byFile.first].push_back(message);
else
{
if (message.type != ERROR)
if (message.getType() != ERROR)
SPF_messages[byFile.first].push_back(message);
else
lastErrors[byFile.first].push_back(message);
@@ -2228,8 +2230,7 @@ void runPass(const int curr_regime, const char *proj_name, const char *folderNam
runPass(GROUP_ACTUAL_AND_REMOTE, proj_name, folderName);
runAnalysis(*project, CALCULATE_STATS_SCHEME, false);
for (auto& elem : allPredictorStats)
__spf_print(1, " stat for file %s: %s\n", elem.first.c_str(), elem.second.to_string().c_str());
//TODO: need to rewrite this to new algo
/*if (!folderName && !consoleMode || predictOn)
runAnalysis(*project, PREDICT_SCHEME, false); */
@@ -2398,7 +2399,7 @@ void runPass(const int curr_regime, const char *proj_name, const char *folderNam
break;
case PARSE_FILES:
{
int err = parseFiles(proj_name, filesCompilationOrder, parseForInlining);
int err = parseFiles(proj_name, filesCompilationOrder, parseForInlining, SPF_messages);
if (err != 0)
throw err;
}
@@ -2667,26 +2668,28 @@ int main(int argc, char **argv)
{
printStackTrace();
printf("exception occurred\n");
json byFileArray = json::array();
for (auto& byFile : SPF_messages)
{
json inFile;
inFile["file"] = byFile.first;
json messages = json::array();
for (auto& message : byFile.second)
{
string toPrint = "";
for (int z = 0; z < message.engMessage.size(); ++z)
toPrint += message.engMessage[z];
string type;
if (message.type == WARR)
type = "WARR";
else if (message.type == ERROR)
type = "ERROR";
else if (message.type == NOTE)
type = "NOTE";
else
type = "UNKN";
printf("%s - [#%d: %s: line %d]: %s\n", type.c_str(), message.group, byFile.first.c_str(), message.line, toPrint.c_str());
message.print(byFile.first);
messages.push_back(message.toJson());
}
inFile["messages"] = messages;
byFileArray.push_back(inFile);
}
json allMessages;
allMessages["allMessage"] = byFileArray;
FILE* outF = fopen("dump_messages.json", "w");
fprintf(outF, "%s", allMessages.dump().c_str());
fclose(outF);
}
deleteAllAllocatedData(withDel);

46
src/Utils/errors.h
View File

@@ -1,5 +1,7 @@
#pragma once
#include <string>
#include "json.hpp"
#ifdef __SPF
#include "dvm.h"
#endif
@@ -143,6 +145,9 @@ enum typeMessage { WARR, ERROR, NOTE };
// 06 "%s clause can be used only once."
// 07 "Variable '%s' can't be used in FILES and EXCEPT clauses at the same time."
// 6000 PARSER GROUP
//
extern int langOfMessages;
struct Messages
{
@@ -164,18 +169,41 @@ public:
engMessage.erase(engMessage.begin() + engMessage.size() - 1);
}
std::wstring toString() const
nlohmann::json toJson() const
{
std::wstring retVal = L"|";
retVal += std::to_wstring((int)type) + L" ";
retVal += std::to_wstring(line) + L" ";
retVal += std::to_wstring(group);
retVal += L"|" + value;
return retVal;
nlohmann::json resVal;
resVal["line"] = line;
resVal["group"] = group;
resVal["value"] = std::string(value.begin(), value.end());
resVal["type"] = (int)type;
return resVal;
}
std::string getString() const { return std::string(engMessage.begin(), engMessage.end()); }
public:
typeMessage getType() const { return type; }
int getLine() const { return line; }
void print(const std::string& file) const
{
std::string toPrint = "";
for (int z = 0; z < engMessage.size(); ++z)
toPrint += engMessage[z];
std::string typeStr;
if (type == WARR)
typeStr = "WARR";
else if (type == ERROR)
typeStr = "ERROR";
else if (type == NOTE)
typeStr = "NOTE";
else
typeStr = "UNKN";
printf("%s - [#%d: %s: line %d]: %s\n", typeStr.c_str(), group, file.c_str(), line, toPrint.c_str());
}
auto getUniqKey() const { return std::make_tuple(type, group, line, value); }
private:
typeMessage type;
int group;
int line;

50
src/Utils/utils.cpp
View File

@@ -30,6 +30,7 @@
#include "../Distribution/Arrays.h"
#include "../DynamicAnalysis/gcov_info.h"
#include "../ParallelizationRegions/ParRegions.h"
#include "json.hpp"
#if __SPF
#include "acc_analyzer.h"
@@ -42,6 +43,7 @@ using std::set;
using std::vector;
using std::string;
using std::wstring;
using json = nlohmann::json;
#if __cplusplus >= 201703L
#include <filesystem>
@@ -427,7 +429,7 @@ static map<string, vector<Messages>> removeCopies(map<string, vector<Messages>>
map<tuple<typeMessage, int, int, wstring>, const Messages*> uniq;
for (auto& message : byFile.second)
{
auto key = make_tuple(message.type, message.group, message.line, message.value);
auto key = message.getUniqKey();
/*string tmp = "";
for (auto& s : message.toString())
tmp += (char)s;
@@ -460,7 +462,7 @@ static void convertGlobalMessagesBuffer(short *&result, int *&resultSize)
bool waschanged = false;
for (auto &message : byFile.second)
{
if (message.line > 0)
if (message.getLine() > 0)
newVal.push_back(message);
else
waschanged = true;
@@ -470,22 +472,30 @@ static void convertGlobalMessagesBuffer(short *&result, int *&resultSize)
byFile.second = newVal;
}
wstring val = L"";
val += std::to_wstring(copySPF_messages.size());
for (auto it = copySPF_messages.begin(); it != copySPF_messages.end(); ++it)
json allMessages = json::array();
for (auto& byFile : copySPF_messages)
{
val += L"|" + to_wstring(it->first.c_str()) + L"|" + std::to_wstring(it->second.size());
for (int k = 0; k < it->second.size(); ++k)
val += it->second[k].toString();
json inFile;
inFile["file"] = byFile.first;
json array = json::array();
for (auto& message : byFile.second)
{
json msg = message.toJson();
array.push_back(msg);
}
inFile["messages"] = array;
allMessages.push_back(inFile);
}
const unsigned len = (unsigned)val.size();
result = new short[len + 1];
allocated.insert(result);
json all;
all["allMessages"] = allMessages;
result[len] = '\0';
for (unsigned i = 0; i < len; ++i)
result[i] = val[i];
const string str = all.dump();
const unsigned len = (unsigned)str.size();
copyStringToShort(result, str);
allocated.insert(result);
resultSize = new int[1];
resultSize[0] = (int)len;
@@ -1686,4 +1696,16 @@ set<string> fillDistributedArrays(const DataDirective& dataDirectives,
for (auto& elem : ret)
distrArrays.insert(shortName ? elem->GetShortName() : elem->GetName());
return distrArrays;
}
void copyStringToShort(short*& result, const string& resVal, bool withEnd)
{
result = new short[resVal.size() + 1];
allocated.insert(result);
for (int i = 0; i < resVal.size(); ++i)
result[i] = resVal[i];
if (withEnd)
result[resVal.size()] = (short)'\0';
}

2
src/Utils/utils.h
View File

@@ -96,3 +96,5 @@ std::vector<std::string> splitAndArgvCreate(const std::string& options);
std::set<DIST::Array*> fillDistributedArraysD(const DataDirective& dataDirectives, const std::map<DIST::Array*, std::tuple<int, std::string, std::string>>& tableOfUniqNamesByArray, const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls, bool onlyCommon = false);
std::set<std::string> fillDistributedArrays(const DataDirective& dataDirectives, const std::map<DIST::Array*, std::tuple<int, std::string, std::string>>& tableOfUniqNamesByArray, const std::map<DIST::Array*, std::set<DIST::Array*>>& arrayLinksByFuncCalls, bool onlyCommon = false, bool shortName = false);
void copyStringToShort(short*& result, const std::string& resVal, bool withEnd = true);

2
src/Utils/version.h
View File

@@ -1,3 +1,3 @@
#pragma once
#define VERSION_SPF "2421"
#define VERSION_SPF "2422"

67
src/VisualizerCalls/get_information.cpp
View File

@@ -218,18 +218,6 @@ static bool tryOpenProjectFile(const char *project)
return ret;
}
static void copyStringToShort(short *&result, const string &resVal, bool withEnd = true)
{
result = new short[resVal.size() + 1];
allocated.insert(result);
for (int i = 0; i < resVal.size(); ++i)
result[i] = resVal[i];
if (withEnd)
result[resVal.size()] = (short)'\0';
}
volatile int passDone = 0;
static volatile int rethrow = 0;
static void runPassesLoop(const vector<passes> &passesToRun, const char *prName, const char *folderNameChar)
@@ -465,14 +453,15 @@ int SPF_ParseFilesWithOrder(void*& context, int winHandler, short* options, shor
{
runPassesForVisualizer(projName, { PARSE_FILES });
string resVal = "";
for (auto& elem : filesCompilationOrder)
{
if (resVal == "")
resVal += elem;
else
resVal += "|" + elem;
}
json filesArray = json::array();
for (auto& file : filesCompilationOrder)
filesArray.push_back(file);
json filesOrder;
filesOrder["allFiles"] = filesArray;
string resVal = filesOrder.dump();
copyStringToShort(result, resVal);
retSize = 0;
}
@@ -1065,7 +1054,6 @@ int SPF_CreateParallelVariant(void*& context, int winHandler, short *options, sh
printf("SAPFOR: set all info done\n");
runPassesForVisualizer(projName, { INSERT_PARALLEL_DIRS }, folderName);
string predictRes = "";
PredictorStats summed;
for (auto &predFile : allPredictorStats)
{
@@ -1081,44 +1069,11 @@ int SPF_CreateParallelVariant(void*& context, int winHandler, short *options, sh
summed.TotalScoreComm += predFile.second.TotalScoreComm;
summed.TotalScoreDist += predFile.second.TotalScoreDist;
summed.TotalScorePar += predFile.second.TotalScorePar;
}
predictRes += summed.to_string();
//predictRes += "|" + to_string((summed.TotalScoreComm != 0 ? 1.0 / summed.TotalScoreComm : 0.0 )+ (double)summed.TotalScorePar * 1000 + (countOfDist == 0 ? -5000 : countOfDist));
if (countOfDist == 0)
predictRes += "|x";
else
predictRes += "|" + to_string(-1 * (summed.ParallelStat.AcrossCount + summed.ParallelStat.RemoteCount + summed.RedistributeCount + summed.RemoteCount));
//predictRes += "|0";
//TODO: need to rewrite to new algo
/*if (folderName == NULL)
{
SpfInterval *mainIterval = getMainInterval(project, intervals);
const int idxBest = mainIterval->getBestTimeIdx();
double speedUpBest = 1;
int procCount = 1;
string topo = "";
if (idxBest != -1 && mainIterval->exec_time != 0)
{
speedUpBest = mainIterval->exec_time / mainIterval->predictedTimes[idxBest];
topo += "[";
for (int z = 0; z < topologies[idxBest].size(); ++z)
{
topo += to_string(topologies[idxBest][z]);
procCount *= topologies[idxBest][z];
if (z != topologies[idxBest].size() - 1)
topo += "x";
}
topo += "]";
}
char buf[256];
sprintf(buf, "%.2f", speedUpBest / procCount * 100.0);
predictRes += "|" + string(buf) + topo;
}
else
predictRes += "|0";*/
string predictRes = summed.toJson().dump();
copyStringToShort(predictorStats, predictRes);
__spf_print(1, " statistic to send: %s\n", predictRes.c_str());
retSize = (int)predictRes.size();
}