diff --git a/src/Predictor/PredictScheme.h b/src/Predictor/PredictScheme.h index b444aa1..35ccf9e 100644 --- a/src/Predictor/PredictScheme.h +++ b/src/Predictor/PredictScheme.h @@ -2,6 +2,7 @@ #include #include "dvm.h" #include "../GraphCall/graph_calls.h" +#include "../Utils/json.hpp" class ParallelStats { @@ -11,10 +12,10 @@ public: RemoteCount = ShadowCount = ReductionCount = AcrossCount = 0; } - int RemoteCount; - int ShadowCount; - int ReductionCount; - int AcrossCount; + int RemoteCount = 0; + int ShadowCount = 0; + int ReductionCount = 0; + int AcrossCount = 0; }; class PredictorStats @@ -27,28 +28,32 @@ public: } ParallelStats ParallelStat; - int ParallelCount; - int RemoteCount; - int RedistributeCount; - int IntervalCount; - int TotalScoreComm; - int TotalScorePar; - int TotalScoreDist; + int ParallelCount = 0; + int RemoteCount = 0; + int RedistributeCount = 0; + int IntervalCount = 0; + int TotalScoreComm = 0; + int TotalScorePar = 0; + int TotalScoreDist = 0; - std::string to_string() + nlohmann::json toJson() { - std::string res = ""; - res += std::to_string(ParallelCount) + "|"; - res += std::to_string(RemoteCount) + "|"; - res += std::to_string(RedistributeCount) + "|"; - res += std::to_string(IntervalCount) + "|"; + nlohmann::json stat; - res += std::to_string(ParallelStat.RemoteCount) + "|"; - res += std::to_string(ParallelStat.ShadowCount) + "|"; - res += std::to_string(ParallelStat.ReductionCount) + "|"; - res += std::to_string(ParallelStat.AcrossCount); + stat["ParallelCount"] = ParallelCount; + stat["RemoteCount"] = RemoteCount; + stat["RedistributeCount"] = RedistributeCount; + stat["IntervalCount"] = IntervalCount; - return res; + stat["PS_RemoteCount"] = ParallelStat.RemoteCount; + stat["PS_ShadowCount"] = ParallelStat.ShadowCount; + stat["PS_ReductionCount"] = ParallelStat.ReductionCount; + stat["PS_AcrossCount"] = ParallelStat.AcrossCount; + + //TODO: need to improve + // (summed.TotalScoreComm != 0 ? 1.0 / summed.TotalScoreComm : 0.0) + (double)summed.TotalScorePar * 1000 + (countOfDist == 0 ? -5000 : countOfDist); + stat["TotalScore"] = -1 * (ParallelStat.RemoteCount + ParallelStat.ShadowCount + ParallelStat.ReductionCount + ParallelStat.AcrossCount); + return stat; } }; diff --git a/src/ProjectManipulation/ParseFiles.cpp b/src/ProjectManipulation/ParseFiles.cpp index cfb469e..d474b85 100644 --- a/src/ProjectManipulation/ParseFiles.cpp +++ b/src/ProjectManipulation/ParseFiles.cpp @@ -387,7 +387,50 @@ static string shiftLines(const string &in, const map &m return newStr; } -static int dumpErrors(const vector& listOfProject, const vector& errors) +static void addMessage(const string& in, const map& mapOfFiles, + const FileInfo* currF, map>& messages, typeMessage type) +{ + int byNum = 0; + + auto it = in.find("on line "); + if (it != string::npos) + it += strlen("on line "); + + int line = 0; + sscanf(in.c_str() + it, "%d", &line); + + auto it1 = in.find("of", it + 1); + if (it1 == string::npos) + return; + it1 += 3; + + string fileN = in.substr(it1, in.find(':', it1) - it1); + auto itF = mapOfFiles.find(fileN); + if (itF != mapOfFiles.end() && itF->second != currF) + { + byNum = itF->second->includesAdded; + if (byNum != 0) + { + if (line - byNum <= 0) + { + //return in; + printInternalError(convertFileName(__FILE__).c_str(), __LINE__); + } + else + line -= byNum; + } + } + + const string newStr = in.substr(0, it) + std::to_string(line) + in.substr(in.find(' ', it + 1)); + + wstring messageE, messageR; + __spf_printToLongBuf(messageE, L"%s", to_wstring(newStr).c_str()); + __spf_printToLongBuf(messageR, L"%s", to_wstring(newStr).c_str()); + + messages[fileN].push_back(Messages(type, line, messageR, messageE, 6000)); +} + +static int dumpErrors(const vector& listOfProject, const vector& errors, map>& messages) { int errorsCount = 0; map mapOfFiles; @@ -420,9 +463,13 @@ static int dumpErrors(const vector& listOfProject, const vector& errors, vector& listOfProject, v return rethrow; } -int parseFiles(const char* proj, vector& filesCompilationOrder, int parseForInlining) +int parseFiles(const char* proj, vector& filesCompilationOrder, int parseForInlining, map>& messages) { FILE* list = fopen(proj, "r"); if (!list) @@ -723,7 +770,7 @@ int parseFiles(const char* proj, vector& filesCompilationOrder, int pars vector errors; int rethrow = parseFiles(errors, listOfProject, filesCompilationOrder, parseForInlining); - int errCount = dumpErrors(listOfProject, errors); + int errCount = dumpErrors(listOfProject, errors, messages); if (rethrow != 0) throw rethrow; @@ -732,6 +779,8 @@ int parseFiles(const char* proj, vector& filesCompilationOrder, int pars void parseFiles(int argc, char** argv) { + map> messages; + bool isInline = false; auto result = splitCommandLineForParse(argv, argc, isInline); if (result.second.size() == 0) diff --git a/src/ProjectManipulation/ParseFiles.h b/src/ProjectManipulation/ParseFiles.h index c9f908b..05fb68a 100644 --- a/src/ProjectManipulation/ParseFiles.h +++ b/src/ProjectManipulation/ParseFiles.h @@ -3,5 +3,5 @@ #include #include -int parseFiles(const char* proj, std::vector& filesCompilationOrder, int parseForInlining); +int parseFiles(const char* proj, std::vector& filesCompilationOrder, int parseForInlining, std::map>& messages); void parseFiles(int argc, char** argv); diff --git a/src/Sapfor.cpp b/src/Sapfor.cpp index 645f7f9..3edcfea 100644 --- a/src/Sapfor.cpp +++ b/src/Sapfor.cpp @@ -64,6 +64,7 @@ #include "DvmhRegions/LoopChecker.h" #include "DvmhRegions/ReadWriteAnalyzer.h" #include "Utils/utils.h" +#include "Utils/json.hpp" #include "Distribution/Array.h" #include "VisualizerCalls/get_information.h" @@ -109,6 +110,7 @@ using namespace std; using std::chrono::high_resolution_clock; using std::chrono::duration_cast; using std::chrono::milliseconds; +using json = nlohmann::json; int PASSES_DONE[EMPTY_PASS]; bool PASSES_DONE_INIT = false; @@ -2053,7 +2055,7 @@ static void findFunctionsToInclude(bool needToAddErrors) SPF_messages[byFile.first].push_back(message); else { - if (message.type != ERROR) + if (message.getType() != ERROR) SPF_messages[byFile.first].push_back(message); else lastErrors[byFile.first].push_back(message); @@ -2228,8 +2230,7 @@ void runPass(const int curr_regime, const char *proj_name, const char *folderNam runPass(GROUP_ACTUAL_AND_REMOTE, proj_name, folderName); runAnalysis(*project, CALCULATE_STATS_SCHEME, false); - for (auto& elem : allPredictorStats) - __spf_print(1, " stat for file %s: %s\n", elem.first.c_str(), elem.second.to_string().c_str()); + //TODO: need to rewrite this to new algo /*if (!folderName && !consoleMode || predictOn) runAnalysis(*project, PREDICT_SCHEME, false); */ @@ -2398,7 +2399,7 @@ void runPass(const int curr_regime, const char *proj_name, const char *folderNam break; case PARSE_FILES: { - int err = parseFiles(proj_name, filesCompilationOrder, parseForInlining); + int err = parseFiles(proj_name, filesCompilationOrder, parseForInlining, SPF_messages); if (err != 0) throw err; } @@ -2667,26 +2668,28 @@ int main(int argc, char **argv) { printStackTrace(); printf("exception occurred\n"); + + json byFileArray = json::array(); for (auto& byFile : SPF_messages) { + json inFile; + inFile["file"] = byFile.first; + + json messages = json::array(); for (auto& message : byFile.second) { - string toPrint = ""; - for (int z = 0; z < message.engMessage.size(); ++z) - toPrint += message.engMessage[z]; - string type; - if (message.type == WARR) - type = "WARR"; - else if (message.type == ERROR) - type = "ERROR"; - else if (message.type == NOTE) - type = "NOTE"; - else - type = "UNKN"; - - printf("%s - [#%d: %s: line %d]: %s\n", type.c_str(), message.group, byFile.first.c_str(), message.line, toPrint.c_str()); + message.print(byFile.first); + messages.push_back(message.toJson()); } + inFile["messages"] = messages; + byFileArray.push_back(inFile); } + json allMessages; + allMessages["allMessage"] = byFileArray; + + FILE* outF = fopen("dump_messages.json", "w"); + fprintf(outF, "%s", allMessages.dump().c_str()); + fclose(outF); } deleteAllAllocatedData(withDel); diff --git a/src/Utils/errors.h b/src/Utils/errors.h index 82e198b..274e779 100644 --- a/src/Utils/errors.h +++ b/src/Utils/errors.h @@ -1,5 +1,7 @@ #pragma once #include +#include "json.hpp" + #ifdef __SPF #include "dvm.h" #endif @@ -143,6 +145,9 @@ enum typeMessage { WARR, ERROR, NOTE }; // 06 "%s clause can be used only once." // 07 "Variable '%s' can't be used in FILES and EXCEPT clauses at the same time." +// 6000 PARSER GROUP +// + extern int langOfMessages; struct Messages { @@ -164,18 +169,41 @@ public: engMessage.erase(engMessage.begin() + engMessage.size() - 1); } - std::wstring toString() const + nlohmann::json toJson() const { - std::wstring retVal = L"|"; - retVal += std::to_wstring((int)type) + L" "; - retVal += std::to_wstring(line) + L" "; - retVal += std::to_wstring(group); - retVal += L"|" + value; - return retVal; + nlohmann::json resVal; + + resVal["line"] = line; + resVal["group"] = group; + resVal["value"] = std::string(value.begin(), value.end()); + resVal["type"] = (int)type; + return resVal; } - std::string getString() const { return std::string(engMessage.begin(), engMessage.end()); } -public: + typeMessage getType() const { return type; } + int getLine() const { return line; } + + void print(const std::string& file) const + { + std::string toPrint = ""; + for (int z = 0; z < engMessage.size(); ++z) + toPrint += engMessage[z]; + + std::string typeStr; + if (type == WARR) + typeStr = "WARR"; + else if (type == ERROR) + typeStr = "ERROR"; + else if (type == NOTE) + typeStr = "NOTE"; + else + typeStr = "UNKN"; + + printf("%s - [#%d: %s: line %d]: %s\n", typeStr.c_str(), group, file.c_str(), line, toPrint.c_str()); + } + + auto getUniqKey() const { return std::make_tuple(type, group, line, value); } +private: typeMessage type; int group; int line; diff --git a/src/Utils/utils.cpp b/src/Utils/utils.cpp index a7d2faf..ea9f073 100644 --- a/src/Utils/utils.cpp +++ b/src/Utils/utils.cpp @@ -30,6 +30,7 @@ #include "../Distribution/Arrays.h" #include "../DynamicAnalysis/gcov_info.h" #include "../ParallelizationRegions/ParRegions.h" +#include "json.hpp" #if __SPF #include "acc_analyzer.h" @@ -42,6 +43,7 @@ using std::set; using std::vector; using std::string; using std::wstring; +using json = nlohmann::json; #if __cplusplus >= 201703L #include @@ -427,7 +429,7 @@ static map> removeCopies(map> map, const Messages*> uniq; for (auto& message : byFile.second) { - auto key = make_tuple(message.type, message.group, message.line, message.value); + auto key = message.getUniqKey(); /*string tmp = ""; for (auto& s : message.toString()) tmp += (char)s; @@ -460,7 +462,7 @@ static void convertGlobalMessagesBuffer(short *&result, int *&resultSize) bool waschanged = false; for (auto &message : byFile.second) { - if (message.line > 0) + if (message.getLine() > 0) newVal.push_back(message); else waschanged = true; @@ -470,22 +472,30 @@ static void convertGlobalMessagesBuffer(short *&result, int *&resultSize) byFile.second = newVal; } - wstring val = L""; - val += std::to_wstring(copySPF_messages.size()); - for (auto it = copySPF_messages.begin(); it != copySPF_messages.end(); ++it) + json allMessages = json::array(); + for (auto& byFile : copySPF_messages) { - val += L"|" + to_wstring(it->first.c_str()) + L"|" + std::to_wstring(it->second.size()); - for (int k = 0; k < it->second.size(); ++k) - val += it->second[k].toString(); + json inFile; + inFile["file"] = byFile.first; + + json array = json::array(); + for (auto& message : byFile.second) + { + json msg = message.toJson(); + array.push_back(msg); + } + inFile["messages"] = array; + allMessages.push_back(inFile); } - const unsigned len = (unsigned)val.size(); - result = new short[len + 1]; - allocated.insert(result); + json all; + all["allMessages"] = allMessages; - result[len] = '\0'; - for (unsigned i = 0; i < len; ++i) - result[i] = val[i]; + const string str = all.dump(); + const unsigned len = (unsigned)str.size(); + + copyStringToShort(result, str); + allocated.insert(result); resultSize = new int[1]; resultSize[0] = (int)len; @@ -1686,4 +1696,16 @@ set fillDistributedArrays(const DataDirective& dataDirectives, for (auto& elem : ret) distrArrays.insert(shortName ? elem->GetShortName() : elem->GetName()); return distrArrays; +} + +void copyStringToShort(short*& result, const string& resVal, bool withEnd) +{ + result = new short[resVal.size() + 1]; + allocated.insert(result); + + for (int i = 0; i < resVal.size(); ++i) + result[i] = resVal[i]; + + if (withEnd) + result[resVal.size()] = (short)'\0'; } \ No newline at end of file diff --git a/src/Utils/utils.h b/src/Utils/utils.h index eb25db1..e96fd1b 100644 --- a/src/Utils/utils.h +++ b/src/Utils/utils.h @@ -96,3 +96,5 @@ std::vector splitAndArgvCreate(const std::string& options); std::set fillDistributedArraysD(const DataDirective& dataDirectives, const std::map>& tableOfUniqNamesByArray, const std::map>& arrayLinksByFuncCalls, bool onlyCommon = false); std::set fillDistributedArrays(const DataDirective& dataDirectives, const std::map>& tableOfUniqNamesByArray, const std::map>& arrayLinksByFuncCalls, bool onlyCommon = false, bool shortName = false); + +void copyStringToShort(short*& result, const std::string& resVal, bool withEnd = true); diff --git a/src/Utils/version.h b/src/Utils/version.h index 495b081..43758a9 100644 --- a/src/Utils/version.h +++ b/src/Utils/version.h @@ -1,3 +1,3 @@ #pragma once -#define VERSION_SPF "2421" +#define VERSION_SPF "2422" diff --git a/src/VisualizerCalls/get_information.cpp b/src/VisualizerCalls/get_information.cpp index 520e929..8aa5102 100644 --- a/src/VisualizerCalls/get_information.cpp +++ b/src/VisualizerCalls/get_information.cpp @@ -218,18 +218,6 @@ static bool tryOpenProjectFile(const char *project) return ret; } -static void copyStringToShort(short *&result, const string &resVal, bool withEnd = true) -{ - result = new short[resVal.size() + 1]; - allocated.insert(result); - - for (int i = 0; i < resVal.size(); ++i) - result[i] = resVal[i]; - - if (withEnd) - result[resVal.size()] = (short)'\0'; -} - volatile int passDone = 0; static volatile int rethrow = 0; static void runPassesLoop(const vector &passesToRun, const char *prName, const char *folderNameChar) @@ -465,14 +453,15 @@ int SPF_ParseFilesWithOrder(void*& context, int winHandler, short* options, shor { runPassesForVisualizer(projName, { PARSE_FILES }); - string resVal = ""; - for (auto& elem : filesCompilationOrder) - { - if (resVal == "") - resVal += elem; - else - resVal += "|" + elem; - } + json filesArray = json::array(); + + for (auto& file : filesCompilationOrder) + filesArray.push_back(file); + + json filesOrder; + filesOrder["allFiles"] = filesArray; + + string resVal = filesOrder.dump(); copyStringToShort(result, resVal); retSize = 0; } @@ -1065,7 +1054,6 @@ int SPF_CreateParallelVariant(void*& context, int winHandler, short *options, sh printf("SAPFOR: set all info done\n"); runPassesForVisualizer(projName, { INSERT_PARALLEL_DIRS }, folderName); - string predictRes = ""; PredictorStats summed; for (auto &predFile : allPredictorStats) { @@ -1081,44 +1069,11 @@ int SPF_CreateParallelVariant(void*& context, int winHandler, short *options, sh summed.TotalScoreComm += predFile.second.TotalScoreComm; summed.TotalScoreDist += predFile.second.TotalScoreDist; summed.TotalScorePar += predFile.second.TotalScorePar; - } - predictRes += summed.to_string(); - //predictRes += "|" + to_string((summed.TotalScoreComm != 0 ? 1.0 / summed.TotalScoreComm : 0.0 )+ (double)summed.TotalScorePar * 1000 + (countOfDist == 0 ? -5000 : countOfDist)); - if (countOfDist == 0) - predictRes += "|x"; - else - predictRes += "|" + to_string(-1 * (summed.ParallelStat.AcrossCount + summed.ParallelStat.RemoteCount + summed.RedistributeCount + summed.RemoteCount)); - //predictRes += "|0"; - - //TODO: need to rewrite to new algo - /*if (folderName == NULL) - { - SpfInterval *mainIterval = getMainInterval(project, intervals); - const int idxBest = mainIterval->getBestTimeIdx(); - double speedUpBest = 1; - int procCount = 1; - string topo = ""; - if (idxBest != -1 && mainIterval->exec_time != 0) - { - speedUpBest = mainIterval->exec_time / mainIterval->predictedTimes[idxBest]; - topo += "["; - for (int z = 0; z < topologies[idxBest].size(); ++z) - { - topo += to_string(topologies[idxBest][z]); - procCount *= topologies[idxBest][z]; - if (z != topologies[idxBest].size() - 1) - topo += "x"; - } - topo += "]"; - } - char buf[256]; - sprintf(buf, "%.2f", speedUpBest / procCount * 100.0); - predictRes += "|" + string(buf) + topo; } - else - predictRes += "|0";*/ + string predictRes = summed.toJson().dump(); copyStringToShort(predictorStats, predictRes); + __spf_print(1, " statistic to send: %s\n", predictRes.c_str()); retSize = (int)predictRes.size(); }