Files
SAPFOR/Sapfor/_src/ProjectManipulation/ParseFiles.cpp
2025-03-25 20:39:29 +03:00

794 lines
25 KiB
C++
Raw Blame History

#include "../Utils/leak_detector.h"
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <cstdint>
#include <string>
#include <fstream>
#include <sstream>
#include <iostream>
#include <vector>
#include <map>
#include <queue>
#include <set>
#include <utility>
#include <string>
#include <assert.h>
#include <locale>
#include <algorithm>
#include <thread>
#include <stack>
#include "../Utils/errors.h"
#include "../Utils/SgUtils.h"
#include "../VisualizerCalls/get_information.h"
#include "../VisualizerCalls/SendMessage.h"
#include "ParseFiles.h"
#include "StdCapture.h"
#include "FileInfo.h"
using namespace std;
extern "C" int parse_file(int argc, char* argv[], char* proj_name);
static void findModuleDeclInProject(const string& name, const vector<string>& files, map<string, string>& modDecls)
{
vector<char*> filesList;
for (int z = 0; z < files.size(); ++z)
filesList.push_back((char*)files[z].c_str());
SgProject* tmpProj = new SgProject(name.c_str(), filesList.data(), files.size());
int numF = tmpProj->numberOfFiles();
for (int z = 0; z < numF; ++z)
{
vector<SgStatement*> modules;
SgFile* currF = &tmpProj->file(z);
string fileName = currF->filename();
convertToLower(fileName);
findModulesInFile(currF, modules);
for (auto& elem : modules)
{
if (string(elem->fileName()) == currF->filename())
{
const string name = elem->symbol()->identifier();
auto it = modDecls.find(name);
if (it != modDecls.end() && it->second != currF->filename())
{
__spf_print(1, "found several module declaration of '%s' in files '%s' and '%s'\n", name.c_str(), it->second.c_str(), currF->filename());
printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
}
else
modDecls.insert(it, make_pair(name, currF->filename()));
}
}
}
InitializeTable();
}
static void createIncludeOrder(vector<string> &toIncl,
const map<string, string>& moduleDelc,
const map<string, set<string>>& modDirectOrder,
set<string> &done,
const string &curr)
{
if (done.find(curr) == done.end())
{
for (auto& elem : modDirectOrder.find(curr)->second)
createIncludeOrder(toIncl, moduleDelc, modDirectOrder, done, elem);
if (done.find(curr) == done.end())
{
toIncl.push_back(moduleDelc.find(curr)->second);
done.insert(curr);
}
}
}
static set<FileInfo*> applyModuleDeclsForFile(FileInfo *forFile, const map<string, FileInfo*> &mapFiles,
const map<string, string>& moduleDelc,
const map<string, set<string>>& mapModuleDeps,
const map<string, set<string>>& modDirectOrder,
vector<string> &optSplited,
bool includeForInline = false)
{
set<FileInfo*> retFilesMod;
auto itF = mapModuleDeps.find(forFile->fileName);
if (itF == mapModuleDeps.end() && !includeForInline)
return retFilesMod;
vector<string> toIncl;
set<string> done;
if (itF != mapModuleDeps.end())
{
for (auto& mod : itF->second)
if (moduleDelc.find(mod) != moduleDelc.end())
createIncludeOrder(toIncl, moduleDelc, modDirectOrder, done, mod);
}
//rewrite files to the next iter of parse
set<FileInfo*> allFiles;
for (auto& incl : toIncl)
{
if (mapFiles.find(incl) == mapFiles.end())
printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
allFiles.insert(mapFiles.find(incl)->second);
}
allFiles.insert(forFile);
set<FileInfo*> toConvert;
int mainStyle = forFile->style;
for (auto& file : allFiles)
if (mainStyle != file->style && file->style != 3)
toConvert.insert(file);
const string mainText = forFile->text;
for (auto& file : toConvert)
file->convertToUniform();
string include = "";
int includeCount = 0;
set<string> included;
for (auto& incl : toIncl)
{
if (included.find(incl) == included.end())
{
include += " include '" + incl + "'\n";
includeCount++;
}
included.insert(incl);
}
vector<string> toInclEnds;
string includeLast = "";
if (includeForInline)
{
//find needed modules first
vector<string> filesWithModules;
for (auto& elem : moduleDelc)
filesWithModules.push_back(elem.second);
for (auto& file : filesWithModules)
{
if (file != forFile->fileName && included.find(file) == included.end())
{
toInclEnds.push_back(file);
included.insert(file);
}
}
for (auto& file : mapFiles)
if (file.second != forFile && included.find(file.second->fileName) == included.end())
toInclEnds.push_back(file.second->fileName);
if (toInclEnds.size())
includeLast += "!SPF SHADOW FILES\n";
for (auto& incl : toInclEnds)
includeLast += " include '" + incl + "'\n";
}
if (includeCount)
include = "!SPF NUM FILES " + std::to_string(includeCount) + "\n" + include;
const string data = include + mainText + includeLast;
__spf_print(1, "include to file %s before\n", forFile->fileName.c_str());
__spf_print(1, "%s", include.c_str());
__spf_print(1, "include to file %s after\n", forFile->fileName.c_str());
__spf_print(1, "%s", includeLast.c_str());
writeFileFromStr(forFile->fileName, data);
/*static string tmp = "tmp__";
static int id = 0;
writeFileFromStr(tmp + to_string(id++) + ".ftn", data);*/
forFile->includesAdded = included.size();
forFile->includes = included;
retFilesMod.insert(forFile);
return retFilesMod;
}
static void restoreOriginalText(const vector<FileInfo>& listOfProject)
{
for (auto& elem : listOfProject)
writeFileFromStr(elem.fileName, elem.text);
fflush(NULL);
}
static inline void restoreOriginalText(const FileInfo& file)
{
writeFileFromStr(file.fileName, file.text);
}
static void checkRetCode(FileInfo& info, const string& errorMessage)
{
if (info.error != 0)
info.lvl++;
if (errorMessage.find("Warning 308") != string::npos)
if (info.error == 0)
info.error = 1;
}
static vector<string> parseList(vector<FileInfo>& listOfProject,
bool needToInclude, bool needToIncludeForInline,
const map<string, set<string>> &mapModuleDeps,
const map<string, string> &moduleDelc,
const map<string, set<string>> &modDirectOrder, bool isFromConsole = false)
{
map<string, FileInfo*> mapFiles;
for (auto& elem : listOfProject)
mapFiles[elem.fileName] = &elem;
vector<string> errors;
int i = 1;
int N = listOfProject.size();
for (auto& elem : listOfProject)
{
sendMessage_progress(std::to_wstring((int)(((double)(i++) / N) * 100)));
string file = elem.fileName;
string options = elem.options;
vector<string> optSplited = splitAndArgvCreate(options);
char** toParse = new char* [optSplited.size() + 1];
for (int z = 0; z < optSplited.size(); ++z)
{
toParse[z] = new char[optSplited[z].size() + 1];
strcpy(toParse[z], optSplited[z].c_str());
}
toParse[optSplited.size()] = new char[file.size() + 1];
strcpy(toParse[optSplited.size()], file.c_str());
if (options.find("-FI") != string::npos)
elem.style = 0;
else if (options.find("-extend_source") != string::npos)
elem.style = 1;
else if (options.find("-FR") != string::npos || options.find("-f90") != string::npos)
elem.style = 2;
else
{ //fdv|f|ftn|for|f90|f95|f03
static set<string> fixed_exts = { "for", "f", "ftn" };
static set<string> free_exts = { "f90", "f95", "f03" };
string ext = OnlyExt(file.c_str());
if (fixed_exts.find(ext) != fixed_exts.end())
elem.style = 0;
else if (free_exts.find(ext) != free_exts.end())
elem.style = 2;
}
for (int z = 0; z < optSplited.size(); ++z)
{
if (optSplited[z] == "-o")
{
if (z + 1 == optSplited.size())
printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
elem.outDepPath = optSplited[z + 1];
break;
}
}
FILE* depPath = fopen(elem.outDepPath.c_str(), "r");
if (depPath && !isFromConsole && !needToIncludeForInline)
{
fclose(depPath);
if (elem.error <= 0)
{
elem.error = 0;
errors.push_back("");
for (int z = 0; z <= optSplited.size(); ++z)
delete toParse[z];
delete[] toParse;
continue;
}
}
#ifdef _WIN32
sendMessage_2lvl(L" <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD> '" + to_wstring(file) + L"'");
#else
sendMessage_2lvl(L" processing file '" + to_wstring(file) + L"'");
#endif
StdCapture::Init();
string errorMessage = "";
try
{
set<FileInfo*> filesModified;
StdCapture::BeginCapture();
if (needToInclude)
filesModified = applyModuleDeclsForFile(&elem, mapFiles, moduleDelc, mapModuleDeps, modDirectOrder, optSplited, needToIncludeForInline);
else if (needToIncludeForInline)
filesModified = applyModuleDeclsForFile(&elem, mapFiles, moduleDelc, mapModuleDeps, modDirectOrder, optSplited, needToIncludeForInline);
int retCode = parse_file(optSplited.size(), toParse, "dvm.proj");
if (needToInclude || needToIncludeForInline)
{
for (auto &elem : filesModified)
restoreOriginalText(*elem);
fflush(NULL);
}
elem.error = retCode;
StdCapture::EndCapture();
errorMessage = StdCapture::GetCapture();
checkRetCode(elem, errorMessage);
}
catch (int err)
{
StdCapture::EndCapture();
errorMessage = StdCapture::GetCapture();
if (needToInclude || needToIncludeForInline)
restoreOriginalText(listOfProject);
}
catch (...)
{
StdCapture::EndCapture();
errorMessage = StdCapture::GetCapture();
if (needToInclude || needToIncludeForInline)
restoreOriginalText(listOfProject);
}
errors.push_back(errorMessage);
for (int z = 0; z <= optSplited.size(); ++z)
delete toParse[z];
delete[] toParse;
createNeededException();
}
return errors;
}
static string shiftLines(const string &in, const map<string, const FileInfo*> &mapOfFiles, const FileInfo* currF)
{
int byNum = 0;
auto it = in.find("on line ");
if (it != string::npos)
it += strlen("on line ");
int d = 0;
sscanf(in.c_str() + it, "%d", &d);
auto it1 = in.find("of", it + 1);
if (it1 == string::npos)
return in;
it1 += 3;
string fileN = in.substr(it1, in.find(':', it1) - it1);
auto itF = mapOfFiles.find(fileN);
if (itF == mapOfFiles.end())
return in;
if (itF->second != currF)
return in;
byNum = itF->second->includesAdded;
if (byNum == 0)
return in;
if (d - byNum <= 0)
{
//return in;
printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
}
else
d -= byNum;
string newStr = in.substr(0, it) + std::to_string(d) + in.substr(in.find(' ', it + 1));
return newStr;
}
static int dumpErrors(const vector<FileInfo>& listOfProject, const vector<string>& errors)
{
int errorsCount = 0;
map<string, const FileInfo*> mapOfFiles;
for (auto& elem : listOfProject)
mapOfFiles[elem.fileName] = &elem;
int z = 0;
for (auto& file : listOfProject)
{
if (errors[z] == "")
{
FILE* ferr = fopen(file.errPath.c_str(), "w");
if (!ferr)
printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
fclose(ferr);
++z;
continue;
}
FILE* ferr = fopen(file.errPath.c_str(), "w");
FILE* fout = fopen(file.outPath.c_str(), "w");
if (!ferr)
printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
if (!fout)
printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
string errS = "", outS = "";
vector<string> splited;
splitString(errors[z], '\n', splited);
for (auto& elem : splited)
{
if (elem.find("Warning 308") != string::npos)
outS += shiftLines(elem, mapOfFiles, &file) + "\n";
else if (elem.find("Error") != string::npos)
{
errS += shiftLines(elem, mapOfFiles, &file) + "\n";
errorsCount++;
}
}
fprintf(fout, "%s", outS.c_str());
fprintf(ferr, "%s", errS.c_str());
fflush(NULL);
fclose(fout);
fclose(ferr);
++z;
}
return errorsCount;
}
static int createMapOfUse(const vector<string>& errors, const vector<FileInfo>& listOfProject, map<string, set<string>> &mapModuleDeps)
{
int changed = 0;
for (int z = 0; z < listOfProject.size(); ++z)
{
if (listOfProject[z].error >= 0)
{
vector<string> splited;
splitString(errors[z], '\n', splited);
for (auto& err : splited)
{
if (err.find("Warning 308") != string::npos && err.find(listOfProject[z].fileName) != string::npos)
{
auto pos = err.find("Unknown module");
if (pos != string::npos)
{
pos += strlen("Unknown module") + 1;
string substr = "";
while (err[pos] != ' ' && pos != err.size())
substr += err[pos++];
mapModuleDeps[listOfProject[z].fileName].insert(substr);
changed++;
}
}
}
}
}
return changed;
}
static map<string, set<string>> createModuleOrder(const map<string, string> &moduleDelc, const map<string, set<string>> &mapModuleDeps)
{
map<string, set<string>> modDirectOrder;
for (auto& elem : moduleDelc)
modDirectOrder[elem.first] = set<string>();
for (auto& elem : moduleDelc)
{
auto itF = mapModuleDeps.find(elem.second);
if (itF != mapModuleDeps.end())
{
for (auto& inFile : itF->second)
{
if (moduleDelc.find(inFile) != moduleDelc.end())
modDirectOrder[elem.first].insert(inFile);
}
}
}
return modDirectOrder;
}
static void printDebug(const map<string, set<string>>& mapModuleDeps, const map<string, set<string>>& modDirectOrder,
const vector<FileInfo>& listOfProject, bool console = false)
{
string toPrint = "MODULE DEPS:\n";
for (auto& elem : mapModuleDeps)
{
toPrint += elem.first + '\n';
for (auto& setEl : elem.second)
toPrint += " " + setEl + '\n';
}
toPrint += "MODULE DIRECT ORDER:\n";
for (auto& elem : modDirectOrder)
{
toPrint += elem.first + '\n';
for (auto& setEl : elem.second)
toPrint += " " + setEl + '\n';
}
toPrint += "FILES LVL:\n";
for (auto& elem : listOfProject)
toPrint += elem.fileName + " " + elem.outDepPath + " lvl = " + std::to_string(elem.lvl) + '\n';
if (console)
printf("%s\n", toPrint.c_str());
__spf_print(1, "%s\n", toPrint.c_str());
}
static void parseFiles(int& iters, vector<string>& errors, vector<FileInfo>& listOfProject,
map<string, set<string>>& mapModuleDeps, map<string, string>& moduleDelc, map<string, set<string>>& modDirectOrder,
int parseForInlining, bool isFromConsole)
{
int changed = 0;
int lastChanged = 0;
const string projName = (parseForInlining == 0) ? "tmp" : "tmp_inl";
do
{
#ifdef _WIN32
sendMessage_1lvl(L"<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> " + std::to_wstring((iters + 1)) + L" <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>");
#else
sendMessage_1lvl(L"running " + std::to_wstring((iters + 1)) + L" iteration of syntax analisys");
#endif
errors = parseList(listOfProject, iters != 0, parseForInlining, mapModuleDeps, moduleDelc, modDirectOrder, isFromConsole);
changed = createMapOfUse(errors, listOfProject, mapModuleDeps);
if (iters != 0)
if (lastChanged <= changed)
break;
createNeededException();
if (changed)
{
vector<string> files;
for (auto& elem : listOfProject)
if (elem.error == 0)
files.push_back(elem.outDepPath);
if (files.size() == 0)
break;
findModuleDeclInProject(projName + std::to_string(iters++), files, moduleDelc);
modDirectOrder = createModuleOrder(moduleDelc, mapModuleDeps);
}
lastChanged = changed;
//printDebug(mapModuleDeps, modDirectOrder, listOfProject);
} while (changed);
}
static vector<string> finalyzeParsing(const vector<FileInfo>& listOfProject,
const map<string, set<string>> mapModuleDeps,
const map<string, string> moduleDelc)
{
vector<string> filesCompilationOrder;
int added = 0;
int iter = 0;
vector<string> files;
while (added != listOfProject.size())
{
for (auto& elem : listOfProject)
{
if (elem.lvl == iter)
{
files.push_back(elem.fileName);
added++;
}
}
++iter;
}
map<string, set<string>> fileDeps;
for (auto& file : files)
{
fileDeps[file] = set<string>();
if (mapModuleDeps.find(file) == mapModuleDeps.end())
continue;
for (auto& dep : mapModuleDeps.at(file))
{
if (moduleDelc.find(dep) == moduleDelc.end())
continue;
fileDeps[file].insert(moduleDelc.at(dep));
}
}
set<string> addedFiles;
added = 0;
while (added != fileDeps.size())
{
for (auto& file : fileDeps)
{
bool depsAdded = true;
for (auto& dep : file.second)
if (addedFiles.find(dep) == addedFiles.end())
depsAdded = false;
if (depsAdded && addedFiles.find(file.first) == addedFiles.end())
{
filesCompilationOrder.push_back(file.first);
addedFiles.insert(file.first);
added++;
}
}
}
__spf_print(1, "files compilation order:\n");
for (auto& file : filesCompilationOrder)
__spf_print(1, " %s\n", file.c_str());
return filesCompilationOrder;
}
static int parseFiles(vector<string>& errors, vector<FileInfo>& listOfProject, vector<string>& filesCompilationOrder,
int parseForInlining, bool isFromConsole = false)
{
int rethrow = 0;
int iters = 0;
map<string, set<string>> mapModuleDeps;
map<string, string> moduleDelc;
map<string, set<string>> modDirectOrder;
try
{
parseFiles(iters, errors, listOfProject, mapModuleDeps, moduleDelc, modDirectOrder, false, isFromConsole);
filesCompilationOrder = finalyzeParsing(listOfProject, mapModuleDeps, moduleDelc);
if (parseForInlining)
parseFiles(iters, errors, listOfProject, mapModuleDeps, moduleDelc, modDirectOrder, true, isFromConsole);
}
catch (int err)
{
rethrow = err;
}
catch (...)
{
rethrow = -1;
}
return rethrow;
}
int parseFiles(const char* proj, vector<string>& filesCompilationOrder, int parseForInlining)
{
FILE* list = fopen(proj, "r");
if (!list)
printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
vector<string> pathSplit;
if (string(proj).find('\\') != string::npos)
splitString(proj, '\\', pathSplit);
else
splitString(proj, '/', pathSplit);
if (pathSplit.size() < 2)
printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
if (pathSplit[pathSplit.size() - 2] != "visualiser_data")
printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
string fullPath = "";
for (int z = 0; z < pathSplit.size() - 2; ++z)
fullPath += pathSplit[z] + "/";
if (fullPath == "")
fullPath = "./";
else
{
//change dir
if (chdir(fullPath.c_str()) != 0)
printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
}
vector<FileInfo> listOfProject;
while (!feof(list))
{
char buf[1024];
if (fgets(buf, 1024, list) == NULL)
continue;
string toAdd = buf;
if (toAdd[toAdd.size() - 1] == '\n')
toAdd = toAdd.erase(toAdd.size() - 1);
string fileNameFixed = "";
auto idx = toAdd.find(fullPath);
if (idx != string::npos)
fileNameFixed = toAdd.substr(idx + fullPath.size());
else
fileNameFixed = (toAdd.substr(0, 2) == "./") ? toAdd.substr(2) : toAdd;
const string optPath = fullPath + "visualiser_data/options/" + fileNameFixed + ".opt";
const string errPath = fullPath + "visualiser_data/options/" + fileNameFixed + ".err";
const string outPath = fullPath + "visualiser_data/options/" + fileNameFixed + ".out";
const string fileText = readFileToStr(toAdd);
FILE* opt = fopen(optPath.c_str(), "r");
if (!opt)
{
__spf_print(1, "can not open path %s\n", optPath.c_str());
printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
}
fgets(buf, 1024, opt);
string toAddOpt = buf;
if (toAddOpt[toAddOpt.size() - 1] == '\n')
toAddOpt = toAddOpt.erase(toAddOpt.size() - 1);
fclose(opt);
listOfProject.push_back(FileInfo(fileNameFixed, toAddOpt, errPath, outPath, "", fileText));
}
fclose(list);
vector<string> errors;
int rethrow = parseFiles(errors, listOfProject, filesCompilationOrder, parseForInlining);
int errCount = dumpErrors(listOfProject, errors);
if (rethrow != 0)
throw rethrow;
return -errCount;
}
void parseFiles(int argc, char** argv)
{
bool isInline = false;
auto result = splitCommandLineForParse(argv, argc, isInline);
if (result.second.size() == 0)
{
printf("Nothing to parse\n");
exit(0);
}
int code = 0;
vector<string> errors;
vector<FileInfo> listOfProject;
string toAddOpt = "";
for (auto& opt : result.first)
toAddOpt += opt + " ";
for (auto& file : result.second)
{
const string fileText = readFileToStr(file);
listOfProject.push_back(FileInfo(file, toAddOpt + "-o " + file + ".dep", "", "", "", fileText, 0));
}
vector<string> filesCompilationOrder;
if (isInline)
printf(" run parsing for inlining\n");
int rethrow = parseFiles(errors, listOfProject, filesCompilationOrder, isInline, true);
if (rethrow == 0)
{
for (auto& err : errors)
{
vector<string> splited;
splitString(err, '\n', splited);
for (auto& elem : splited)
{
if (elem.find("Error") != string::npos)
{
printf("%s\n", elem.c_str());
code++;
}
}
}
}
if (code == 0 && rethrow == 0)
{
FILE* proj = fopen("dvm.proj", "w");
if (proj == NULL)
printInternalError(convertFileName(__FILE__).c_str(), __LINE__);
for (auto& file : result.second)
fprintf(proj, "%s.dep\n", file.c_str());
printf("Parsing was completed successfully\n");
}
else
printf("Parsing was completed with errors, throw code %d, errors count %d\n", rethrow, code);
exit(0);
}