Clean up

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "projects/dvm"]
 	path = projects/dvm
 	url = https://dvmguest:dvmguest@dvm.keldysh.ru/dvm-system/dvm
+[submodule "projects/libpredictor"]
+	path = projects/libpredictor
+	url = https://dvmguest:dvmguest@dvm.keldysh.ru/sapfor/dvm-perfm

CMakeLists.txt

@@ -13,16 +13,18 @@ add_definitions("-D YYDEBUG")
 
 set(CMAKE_CXX_STANDARD 17)
 
-set(fdvm_include   projects/dvm/fdvmh/include/fdvmh/)
-set(fdvm_sources   projects/dvm/fdvmh/tools/fdvmh/)
-set(sage_include_1 projects/dvm/fdvmh/include/sage/lib/)
-set(sage_include_2 projects/dvm/fdvmh/include/sage/h/)
-set(libdb_sources  projects/dvm/fdvmh/lib/sage/db/)
-set(sage_sources   projects/dvm/fdvmh/lib/sage/sage/)
-set(sagepp_sources projects/dvm/fdvmh/lib/sage/sage++/)
-set(parser_sources projects/dvm/fdvmh/tools/parser/)
-set(pppa_sources   projects/dvm/pppa/src/)
-set(zlib_sources   projects/dvm/third-party/Zlib/)
+set(fdvm_include    projects/dvm/fdvmh/include/fdvmh/)
+set(fdvm_sources    projects/dvm/fdvmh/tools/fdvmh/)
+set(sage_include_1  projects/dvm/fdvmh/include/sage/lib/)
+set(sage_include_2  projects/dvm/fdvmh/include/sage/h/)
+set(libdb_sources   projects/dvm/fdvmh/lib/sage/db/)
+set(sage_sources    projects/dvm/fdvmh/lib/sage/sage/)
+set(sagepp_sources  projects/dvm/fdvmh/lib/sage/sage++/)
+set(parser_sources  projects/dvm/fdvmh/tools/parser/)
+set(pppa_sources    projects/dvm/pppa/src/)
+set(zlib_sources    projects/dvm/third-party/Zlib/)
+set(libpred_sources projects/libpredictor/src/)
+set(libpred_include projects/libpredictor/include/)
 
 include_directories(src)
 #Sage lib includes
@@ -33,6 +35,8 @@ include_directories(${sage_include_2})
 include_directories(${zlib_sources}/include)
 #PPPA includes
 include_directories(${pppa_sources})
+#Libpredictor includes
+include_directories(${libpred_include})
 
 set(PR_PARAM src/ProjectParameters/projectParameters.cpp
              src/ProjectParameters/projectParameters.h)
@@ -312,6 +316,15 @@ set(MAIN src/Sapfor.cpp
 
 set(PREDICTOR src/Predictor/PredictScheme.cpp
               src/Predictor/PredictScheme.h)
+              
+set(LIBPREDICTOR  ${libpred_sources}/cluster.cpp
+                  ${libpred_sources}/predictor.cpp
+                  ${libpred_sources}/transfer.cpp
+                  ${libpred_sources}/utils.cpp
+                  ${libpred_include}/libpredict/predictor.h
+                  ${libpred_include}/internal/cluster.h
+                  ${libpred_include}/internal/transfer.h
+                  ${libpred_include}/internal/utils.h)
 
 set(PROJ_MAN src/ProjectManipulation/ParseFiles.cpp
              src/ProjectManipulation/ParseFiles.h
@@ -401,6 +414,7 @@ set(SOURCE_EXE
     ${VS_CALLS}
     ${MAIN}
     ${PREDICTOR}
+    ${LIBPREDICTOR}
     ${PARSER}
     ${PPPA}
     ${ZLIB}
@@ -455,7 +469,8 @@ source_group (VisualizerCalls FILES ${VS_CALLS})
 source_group (VisualizerCalls\\GraphLayout FILES ${GR_LAYOUT})
 
 source_group (_SapforMain FILES ${MAIN})
-source_group (Predictor FILES ${PREDICTOR})
+source_group (Predictor\\Analyzer FILES ${PREDICTOR})
+source_group (Predictor\\Library FILES ${LIBPREDICTOR})
 source_group (Parser FILES ${PARSER})
 source_group (PPPA\\PPPA FILES ${PPPA})
 source_group (PPPA\\ZLib FILES ${ZLIB})

src/DvmhRegions/DvmhRegionInserter.cpp

@@ -834,19 +834,21 @@ static string getInterfaceBlock(SgStatement* func, const FuncParam& pars)
     auto copy = duplicateProcedure(func, NULL, false, false, false, true);
     const set<string> idents(pars.identificators.begin(), pars.identificators.end());
 
+    bool need = (func->symbol()->identifier() == string("bl182"));
+
     //remove all exec
     SgStatement* st = copy->lexNext();
     SgStatement* last = copy->lastNodeOfStmt();
     vector<SgStatement*> toExtract;
+
     while (st != last)
     {
         if (isDVM_stat(st) || isSPF_stat(st))
         {
             if (st->variant() != ACC_ROUTINE_DIR)
             {
-                SgStatement* next = st->lexNext();
-                st->extractStmt();
-                st = next;
+                toExtract.push_back(st);
+                st = st->lexNext();
             }
             else
                 st = st->lexNext();
@@ -868,6 +870,7 @@ static string getInterfaceBlock(SgStatement* func, const FuncParam& pars)
     while (st != last)
     {
         const int var = st->variant();
+
         if (var == VAR_DECL
             || var == VAR_DECL_90
             || var == DIM_STAT
@@ -877,9 +880,8 @@ static string getInterfaceBlock(SgStatement* func, const FuncParam& pars)
             bool empty = filterFromList(st, idents);
             if (empty)
             {
-                SgStatement* next = st->lexNext();
                 toExtract.push_back(st);
-                st = next;
+                st = st->lexNext();
                 continue;
             }
         }

src/Predictor/PredictScheme.cpp

@@ -14,13 +14,13 @@
 
 #include "dvm.h"
 #include "../DynamicAnalysis/gcov_info.h"
+#include "../DynamicAnalysis/gCov_parser_func.h"
 #include "PredictScheme.h"
 #include "../Utils/SgUtils.h"
 #include "../DirectiveProcessing/directive_parser.h"
 #include "../Distribution/DvmhDirective.h"
 #include "../GraphLoop/graph_loops_func.h"
 #include "../ExpressionTransform/expr_transform.h"
-#include "../DirectiveProcessing/directive_parser.h"
 #include "../LoopAnalyzer/loop_analyzer.h"
 #include "../CFGraph/CFGraph.h"
 
@@ -208,27 +208,31 @@ void calculateStatsForPredictor(const map<string, vector<FuncInfo*>>& allFuncInf
                 uint64_t paralle_exec = 0;
                 uint64_t lines_count = 0;
 
-                if (st->variant() == DVM_PARALLEL_ON_DIR) 
+                if (st->variant() == DVM_PARALLEL_ON_DIR)
                 {
                     auto loop = st->lexNext();
                     checkNull(loop, convertFileName(__FILE__).c_str(), __LINE__);
                     if (loop->variant() != FOR_NODE)
                         printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
 
-                    calculateForParallelLoop(loop, gcov, paralle_exec, lines_count);
-                    st = loop->lastNodeOfStmt();
+                    if (__gcov_doesThisLineExecuted(byFile.first, loop->lineNumber()))
+                    {
+                        calculateForParallelLoop(loop, gcov, paralle_exec, lines_count);
+                        st = loop->lastNodeOfStmt();
 
-                    parallel_exec_count += paralle_exec;
-                    count_of_parallel_lines += lines_count;
+                        parallel_exec_count += paralle_exec;
+                        count_of_parallel_lines += lines_count;
 
-                    __spf_print(1, "    PAR LOOP [%d %s] total exec %llu, total exec lines %llu, avg %.16e\n", 
-                                    loop->lineNumber(), byFile.first.c_str(), paralle_exec, lines_count, paralle_exec / (double)lines_count);
+                        __spf_print(1, "    PAR LOOP [%d %s] total exec %llu, total exec lines %llu, avg %.16e\n",
+                            loop->lineNumber(), byFile.first.c_str(), paralle_exec, lines_count, paralle_exec / (double)lines_count);
+                    }
                 }
             }
 
             for (auto st = stat->lexNext(); st != stat->lastNodeOfStmt(); st = st->lexNext()) 
             {
-                if (!isSgExecutableStatement(st) || isDVM_stat(st) || isSPF_stat(st))
+                if (!isSgExecutableStatement(st) || isDVM_stat(st) || isSPF_stat(st) ||
+                    !__gcov_doesThisLineExecuted(byFile.first, st->lineNumber()))
                     continue;
 
                 int line = st->lineNumber();
@@ -428,6 +432,7 @@ static void parallelDir(const map<DIST::Array*, int>& byPos, SgExpression* spec,
     parallel["loops_count"] = loopSymbs.size();
 
     SgStatement* loop = isSgForStmt(st->lexNext());
+
     if (loop == NULL)
         printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
     SgStatement* lastNode = loop->lastNodeOfStmt();
@@ -444,11 +449,13 @@ static void parallelDir(const map<DIST::Array*, int>& byPos, SgExpression* spec,
         loop = loop->lexNext();
     }
 
-    for (int z = execs.size() - 1; z > 0; --z)
-        execs[z] /= execs[z - 1];
+    for (int z = execs.size() - 1; z > 0; --z) 
+        if (execs[z - 1] != 0)
+            execs[z] /= execs[z - 1];
 
     auto& info = getInfo(before, gcov);
-    execs[0] /= info.getExecutedCount();
+    if (info.getExecutedCount() && loopSymbs.size() > 1)
+        execs[0] /= info.getExecutedCount();
 
     parallel["iterations_count"] = execs;
 
@@ -587,6 +594,9 @@ void parseDvmDirForPredictor(const map<tuple<int, string, string>, pair<DIST::Ar
 
             for (auto st = stat->lexNext(); st != stat->lastNodeOfStmt(); st = st->lexNext())
             {
+                if (!__gcov_doesThisLineExecuted(byFile.first, st->lineNumber()))
+                    continue;
+
                 SgExpression* list;
                 SgExpression* dup;
                 auto line = 0;
@@ -594,7 +604,10 @@ void parseDvmDirForPredictor(const map<tuple<int, string, string>, pair<DIST::Ar
                 switch (st->variant())
                 {
                 case DVM_PARALLEL_ON_DIR:
-                    parallelDir(byPos, st->expr(2), st->expr(0)->symbol(), st->expr(0)->lhs(), st, st->expr(1), gcov, directives, commonBlocks, allFuncInfo);
+                    parallelDir(byPos, st->expr(2), 
+                                st->expr(0) ? st->expr(0)->symbol() : NULL, 
+                                st->expr(0) ? st->expr(0)->lhs() : NULL, 
+                                st, st->expr(1), gcov, directives, commonBlocks, allFuncInfo);
                     break;
                 case DVM_VAR_DECL: // TODO
                 {
@@ -613,7 +626,8 @@ void parseDvmDirForPredictor(const map<tuple<int, string, string>, pair<DIST::Ar
                         list = st->expr(0);
                         while (list)
                         {
-                            directives["align"].push_back(parseAlign(byPos, list->lhs()->symbol(), type->rhs()->symbol(), type->lhs(), type->rhs()->lhs(), st->lineNumber()));
+                            if (type->lhs()) // if ALIGN A(...) with B(...)
+                                directives["align"].push_back(parseAlign(byPos, list->lhs()->symbol(), type->rhs()->symbol(), type->lhs(), type->rhs()->lhs(), st->lineNumber()));
                             list = list->rhs();
                         }
                     }
@@ -625,6 +639,9 @@ void parseDvmDirForPredictor(const map<tuple<int, string, string>, pair<DIST::Ar
                 case DVM_ALIGN_DIR:
                     directives["align"].push_back(parseAlign(byPos, st->expr(0)->lhs()->symbol(), st->expr(2)->symbol(), st->expr(1), st->expr(2)->lhs(), st->lineNumber()));
                     break;
+                case DVM_REALIGN_DIR:
+                    directives["realign"].push_back(parseAlign(byPos, st->expr(0)->lhs()->symbol(), st->expr(2)->symbol(), st->expr(1), st->expr(2)->lhs(), st->lineNumber()));
+                    break;
                 case DVM_SHADOW_DIR:
                     //dirs << "1;" << "SHADOW;" << st->expr(0)->unparse() << "(" << st->expr(1)->unparse() << ");\n";
                     break;

src/Sapfor.cpp

@@ -331,7 +331,7 @@ static string unparseProjectIfNeed(SgFile* file, const int curr_regime, const bo
             unparseToBuf = removeIncludeStatsAndUnparse(file, file_name, fout_name.c_str(), allIncludeFiles, out_free_form == 1, moduleUsesByFile, 
                                                         moduleDecls, getObjectForFileFromMap(file_name, exctactedModuleStats), toString, false, true);
             auto itI = filesToInclude.find(file_name);
-            for (auto& incl : itI->second)
+            for (auto& [_, incl] : itI->second)
                 if (allIncludeFiles.find(incl) != allIncludeFiles.end())
                     allIncludeFiles.erase(incl);
         }
@@ -827,20 +827,27 @@ static bool runAnalysis(SgProject &project, const int curr_regime, const bool ne
         {
             auto fileIt = includeDependencies.find(file_name);
             if (fileIt == includeDependencies.end())
-                fileIt = includeDependencies.insert(fileIt, make_pair(file_name, set<string>()));
+                fileIt = includeDependencies.insert(fileIt, make_pair(file_name, vector<pair<int, string>>()));
 
             set<string> modFiles;
             for (auto& elem : moduleDecls)
                 modFiles.insert(elem.second);
 
-            for (SgStatement *first = file->firstStatement(); first; first = first->lexNext())
+            SgStatement* lastFromFile = NULL;
+            for (SgStatement *st = file->firstStatement(); st; st = st->lexNext())
             {
-                if (strcmp(file_name, first->fileName()))
+                if (st->variant() == MODULE_STMT && modFiles.find(st->fileName()) != modFiles.end())
+                    st = st->lastNodeOfStmt();
+                else
                 {
-                    if (first->variant() == MODULE_STMT && modFiles.find(first->fileName()) != modFiles.end())
-                        first = first->lastNodeOfStmt();
+                    if (strcmp(file_name, st->fileName()))
+                    {
+                        if (lastFromFile == NULL)
+                            printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
+                        fileIt->second.push_back(make_pair(lastFromFile->lineNumber(), st->fileName()));
+                    }
                     else
-                        fileIt->second.insert(first->fileName());
+                        lastFromFile = st;
                 }
             }
 
@@ -1596,9 +1603,9 @@ static bool runAnalysis(SgProject &project, const int curr_regime, const bool ne
                     set<string> includedToThisFile;
                     if (itDep != includeDependencies.end())
                     {
-                        for (auto &inclDep : itDep->second)
+                        for (auto& [_, incl] : itDep->second)
                         {
-                            auto comm = commentsToInclude.find(inclDep);
+                            auto comm = commentsToInclude.find(incl);
                             if (comm != commentsToInclude.end())
                                 for (auto &allComm : comm->second)
                                     includedToThisFile.insert(allComm.second.begin(), allComm.second.end());

src/SapforData.h

@@ -85,7 +85,7 @@ std::map<std::string, std::map<int, std::set<std::string>>> commentsToInclude;
 //
 
 //for INSERT_INCLUDES
-std::map<std::string, std::set<std::string>> filesToInclude; // file -> includes
+std::map<std::string, std::vector<std::pair<int, std::string>>> filesToInclude; // file -> includes [nearest line, include]
 //
 
 //for PASSES DEPENDENSIES
@@ -96,7 +96,7 @@ std::set<passes> passesIgnoreStateDone;
 //for files info
 std::map<std::string, int> lineInfo; // file -> lines count
 std::map<std::string, std::pair<std::set<int>, std::set<int>>> dirsInfo; // file -> dirs <lines SPF, lines DVM> count
-std::map<std::string, std::set<std::string>> includeDependencies; // file -> includes
+std::map<std::string, std::vector<std::pair<int, std::string>>> includeDependencies; // file -> includes [nearest line, include]
 std::vector<std::string> filesCompilationOrder; // order of files for unite to one file
 std::map<std::string, std::map<SgStatement*, std::vector<SgStatement*>>> exctactedModuleStats; // file -> hided excluded modules
 //

src/Utils/PassManager.h

@@ -309,6 +309,8 @@ void InitPassesDependencies(map<passes, vector<passes>> &passDepsIn, set<passes>
 
     Pass(REMOVE_OMP_DIRS) <= Pass(REMOVE_OMP_DIRS_TRANSFORM);
 
+    Pass(VERIFY_INCLUDES) <= Pass(RENAME_INLCUDES);
+
     list({ CALL_GRAPH2, REVERT_SUBST_EXPR_RD }) <= Pass(REMOVE_DEAD_CODE);
     list({ REMOVE_DEAD_CODE, CONVERT_LOOP_TO_ASSIGN, RESTORE_LOOP_FROM_ASSIGN }) <= Pass(REMOVE_DEAD_CODE_AND_UNPARSE);
 

src/Utils/version.h

@@ -1,3 +1,3 @@
 #pragma once
 
-#define VERSION_SPF "2406"
+#define VERSION_SPF "2414"

src/VisualizerCalls/get_information.cpp

@@ -1308,13 +1308,12 @@ int SPF_GetIntrinsics(void*& context, short *&result)
     return (int)resVal.size() + 1;
 }
 
-extern map<string, set<string>> includeDependencies;
+extern map<string, vector<pair<int, string>>> includeDependencies;
 int SPF_GetIncludeDependencies(void*& context, int winHandler, short *options, short *projName, short *&result, short*& output, int*& outputSize,
                                short*& outputMessage, int*& outputMessageSize)
 {
     MessageManager::clearCache();
     MessageManager::setWinHandler(winHandler);
-    string resVal = "";
 
     setOptions(options);
     int retSize = 0;
@@ -1322,23 +1321,28 @@ int SPF_GetIncludeDependencies(void*& context, int winHandler, short *options, s
     {
         runPassesForVisualizer(projName, { BUILD_INCLUDE_DEPENDENCIES });
 
-        int i = 0;
-        for (auto &deps : includeDependencies)
+        json inc_array = json::array();
+        for (const auto& deps : includeDependencies)
         {
-            if (i != 0)
-                resVal += "@";
-            resVal += deps.first + "@";
-            int k = 0;
-            for (auto &incl : deps.second)
+            json includes;
+            includes["file"] = deps.first;
+
+            json array = json::array();
+            for (const auto& [line, incl] : deps.second)
             {
-                if (k != 0)
-                    resVal += "|";
-                resVal += incl;
-                ++k;
+                json elem;
+                elem["line"] = line;
+                elem["dependencyFileName"] = incl;
+                array.push_back(elem);
             }
-            ++i;
+            includes["includes"] = array;
+            inc_array.push_back(includes);
         }
 
+        json allIncludes;
+        allIncludes["allIncludes"] = inc_array;
+        string resVal = allIncludes.dump();
+
         copyStringToShort(result, resVal);
         retSize = (int)resVal.size() + 1;
     }
@@ -1460,19 +1464,33 @@ int SPF_GetFileLineInfo(void*& context, int winHandler, short *options, short *p
     {
         runPassesForVisualizer(projName, { FILE_LINE_INFO });
 
-        string resVal = "";
-        for (auto it = lineInfo.begin(); it != lineInfo.end(); ++it)
+        json metric_array = json::array();
+        for (const auto& fileInfo : lineInfo)
         {
-            if (it != lineInfo.begin())
-                resVal += "@";
+            json fileMetric;
 
-            auto itD = dirsInfo.find(it->first);
-            if (itD == dirsInfo.end())
-                resVal += it->first + "@" + to_string(it->second) + "_0_0";
+            fileMetric["file"] = fileInfo.first;
+            fileMetric["linesCount"] = fileInfo.second;
+            
+            auto it = dirsInfo.find(fileInfo.first);
+            if (it == dirsInfo.end())
+            {
+                fileMetric["numSPF"] = 0;
+                fileMetric["numDVM"] = 0;
+            }
             else
-                resVal += it->first + "@" + to_string(it->second) + "_" + to_string(itD->second.first.size()) + "_" + to_string(itD->second.second.size());
+            {
+                fileMetric["numSPF"] = (int)it->second.first.size();
+                fileMetric["numDVM"] = (int)it->second.second.size();
+            }
+
+            metric_array.push_back(fileMetric);
         }
 
+        json allMetrics;
+        allMetrics["allMetrics"] = metric_array;
+        string resVal = allMetrics.dump();
+
         copyStringToShort(result, resVal);
         retSize = (int)resVal.size() + 1;
     }
@@ -2171,38 +2189,44 @@ int SPF_InlineProcedures(void*& context, int winHandler, short* options, short*
 }
 
 
-extern map<string, set<string>> filesToInclude;
-int SPF_InsertIncludesPass(void*& context, int winHandler, short *options, short *projName, short *folderName, char *filesToInclude,
+extern map<string, vector<pair<int, string>>> filesToInclude;
+int SPF_InsertIncludesPass(void*& context, int winHandler, short *options, short *projName, short *folderName, char *visFilesToInclude,
                            short *&output, int *&outputSize, short *&outputMessage, int *&outputMessageSize)
 {
     MessageManager::clearCache();
     MessageManager::setWinHandler(winHandler);
 
-    if (filesToInclude == NULL)
+    if (visFilesToInclude == NULL)
         return -2;
 
     vector<string> splited;
     //printf("%s\n", conv);
-    splitString(filesToInclude, '|', splited);
+    splitString(visFilesToInclude, '|', splited);
 
     if (splited.size() == 0)
         return -3;
 
-    ::filesToInclude.clear();
-    for (int i = 0; i < splited.size(); ++i)
+    filesToInclude.clear();
+    unsigned i = 0;
+    while (i < splited.size())
     {
-        string file = splited[i];
+        string file = splited[i++];
         int num = 0;
-        if (sscanf(splited[i + 1].c_str(), "%d", &num) == -1)
-            return -3;
+        if (sscanf(splited[i++].c_str(), "%d", &num) == -1)
+            return -4;
 
         __spf_print(1, "file = %s:\n", file.c_str());
-        for (int k = i + 2; k < i + 2 + num; ++k)
+        for (int k = i; k < i + 2 * num; k += 2)
         {
-            ::filesToInclude[file].insert(splited[k]);
-            __spf_print(1, "    include = %s\n", splited[k].c_str());
+            int line = 0;
+            if (sscanf(splited[k].c_str(), "%d", &line) == -1)
+                return -5;
+
+            auto pair = make_pair(line, splited[k + 1]);
+            filesToInclude[file].push_back(pair);
+            __spf_print(1, "    include = [%d %s]\n", pair.first, pair.second.c_str());
         }
-        i += 1 + num;
+        i += 2 * num;
     }
     return simpleTransformPass(INSERT_INCLUDES, options, projName, folderName, output, outputSize, outputMessage, outputMessageSize);
 }