497 lines
23 KiB
Java
497 lines
23 KiB
Java
package TestingSystem.DVM;
|
||
import Common.Constants;
|
||
import Common.Current;
|
||
import Common.Global;
|
||
import Common.Utils.Utils;
|
||
import GlobalData.Machine.Machine;
|
||
import GlobalData.Machine.MachineType;
|
||
import GlobalData.RemoteFile.RemoteFile;
|
||
import GlobalData.Tasks.TaskState;
|
||
import GlobalData.User.User;
|
||
import ProjectData.Files.ProjectFile;
|
||
import ProjectData.LanguageName;
|
||
import Repository.Server.ServerCode;
|
||
import TestingSystem.Common.TasksPackageState;
|
||
import TestingSystem.Common.TestingPlanner;
|
||
import TestingSystem.DVM.DVMPackage.DVMPackage;
|
||
import TestingSystem.DVM.DVMTasks.DVMCompilationTask;
|
||
import TestingSystem.DVM.DVMTasks.DVMRunTask;
|
||
import TestingSystem.DVM.DVMTasks.DVMTask;
|
||
import Visual_DVM_2021.Passes.All.UnzipFolderPass;
|
||
import Visual_DVM_2021.UI.Interface.Loggable;
|
||
import javafx.util.Pair;
|
||
import org.apache.commons.io.FileUtils;
|
||
|
||
import java.io.File;
|
||
import java.io.FileFilter;
|
||
import java.net.InetAddress;
|
||
import java.nio.charset.Charset;
|
||
import java.nio.file.Paths;
|
||
import java.util.*;
|
||
public class MachineQueueSupervisor extends TestingPlanner<DVMPackage> {
|
||
File supervisorHome = null;
|
||
Machine machine = null;
|
||
User user = null;
|
||
boolean local;
|
||
RemoteFile packageRemoteWorkspace = null;
|
||
File packageLocalWorkspace = null;
|
||
String serverName = "";
|
||
//----
|
||
public MachineQueueSupervisor(String... args) {
|
||
Global.isWindows = System.getProperty("os.name").startsWith("Windows");
|
||
//---
|
||
String machineAddress = args[0];
|
||
int machinePort = Integer.parseInt(args[1]);
|
||
String userName = args[2];
|
||
String userPassword = args[3];
|
||
String userWorkspace = args[4];
|
||
String testingSystemRoot = args[5];
|
||
serverName = args[6];
|
||
supervisorHome = new File(Global.Home); //при инициализации это текущая папка.
|
||
//---
|
||
Global.Log = new Loggable() {
|
||
@Override
|
||
public String getLogHomePath() {
|
||
return supervisorHome.getAbsolutePath();
|
||
}
|
||
@Override
|
||
public String getLogName() {
|
||
return Current.mode.toString();
|
||
}
|
||
};
|
||
Global.Log.ClearLog();
|
||
//--
|
||
Global.Home = testingSystemRoot;
|
||
Global.CheckTestingSystemDirectories();
|
||
System.out.println(Global.TestsDirectory.getAbsolutePath());
|
||
//---
|
||
machine = new Machine(machineAddress, machineAddress, machinePort, MachineType.Server);
|
||
user = new User(userName, userPassword, userWorkspace);
|
||
CheckLocal();
|
||
//---
|
||
Print("machineAddress=" + Utils.Brackets(machineAddress));
|
||
Print("machinePort=" + Utils.Brackets(String.valueOf(machinePort)));
|
||
Print("userName=" + Utils.Brackets(userName));
|
||
Print("userPassword=" + Utils.Brackets(userPassword));
|
||
Print("userWorkspace=" + Utils.Brackets(userWorkspace));
|
||
Print("root=" + Utils.Brackets(Global.Home));
|
||
Print("local=" + local);
|
||
Print("serverName=" + serverName);
|
||
Print("=====");
|
||
//----
|
||
File started = new File(supervisorHome, Constants.STARTED);
|
||
try {
|
||
FileUtils.writeStringToFile(started, "");
|
||
} catch (Exception ex) {
|
||
ex.printStackTrace();
|
||
Finalize("Can not start");
|
||
}
|
||
}
|
||
void CheckLocal() {
|
||
local = false;
|
||
try {
|
||
InetAddress address = InetAddress.getByName(machine.address);
|
||
InetAddress localAddress = InetAddress.getByName("alex-freenas.ddns.net");
|
||
Print("machine ip=" + Utils.Brackets(address.getHostAddress()));
|
||
Print("server ip=" + Utils.Brackets(localAddress.getHostAddress()));
|
||
local = localAddress.getHostAddress().equals(address.getHostAddress());
|
||
//todo в этом случае отдельный режим без ssh
|
||
} catch (Exception ex) {
|
||
Global.Log.PrintException(ex);
|
||
}
|
||
}
|
||
public String getPlanner() {
|
||
return String.join("/", user.workspace, "modules", "planner");
|
||
}
|
||
@Override
|
||
protected boolean Connect() {
|
||
if (user.connection == null) {
|
||
try {
|
||
user.connection = new UserConnection(machine, user);
|
||
Print("Соединение c " + machine.getURL() + " " + user.login + " успешно установлено.");
|
||
} catch (Exception ex) {
|
||
Print(ex.toString());
|
||
user.connection = null;
|
||
Print("Не удалось установить соединение.");
|
||
}
|
||
}
|
||
return user.connection != null;
|
||
}
|
||
@Override
|
||
protected void Disconnect() {
|
||
if (user.connection != null) {
|
||
user.connection.Disconnect();
|
||
user.connection = null;
|
||
Print("Соединение c " + machine.getURL() + " " + user.login + " сброшено.");
|
||
}
|
||
}
|
||
@Override
|
||
protected void Print(String message) {
|
||
try {
|
||
if (isPrintOn()) {
|
||
// System.out.println(message);
|
||
Global.Log.Print(message);
|
||
}
|
||
} catch (Exception ex) {
|
||
ex.printStackTrace();
|
||
}
|
||
}
|
||
void Finalize(String reason) {
|
||
Print(reason);
|
||
File stateFile = new File(supervisorHome, Constants.ABORTED);
|
||
try {
|
||
FileUtils.writeStringToFile(stateFile, reason);
|
||
} catch (Exception ex) {
|
||
ex.printStackTrace();
|
||
}
|
||
System.exit(0);
|
||
}
|
||
//--
|
||
//Получить ид тестов и их папки на сервере.
|
||
LinkedHashMap<Integer, File> getTestsFromJson() {
|
||
LinkedHashMap<Integer, File> res = new LinkedHashMap<>();
|
||
for (DVMCompilationTask task : testingPackage.package_json.compilationTasks) {
|
||
if (!res.containsKey(task.test_id)) {
|
||
res.put(task.test_id, Paths.get(Global.TestsDirectory.getAbsolutePath(), String.valueOf(task.test_id)).toFile());
|
||
}
|
||
}
|
||
return res;
|
||
}
|
||
static LinkedHashMap<LanguageName, Vector<ProjectFile>> getTestPrograms(File test) {
|
||
LinkedHashMap<LanguageName, Vector<ProjectFile>> res = new LinkedHashMap<>();
|
||
//--
|
||
res.put(LanguageName.fortran, new Vector<>());
|
||
res.put(LanguageName.c, new Vector<>());
|
||
res.put(LanguageName.cpp, new Vector<>());
|
||
//--
|
||
File[] files = test.listFiles(new FileFilter() {
|
||
@Override
|
||
public boolean accept(File pathname) {
|
||
return pathname.isFile();
|
||
}
|
||
});
|
||
if (files != null) {
|
||
for (File file : files) {
|
||
ProjectFile projectFile = new ProjectFile(new File(file.getName()));
|
||
if (projectFile.isNotExcludedProgram()) res.get(projectFile.languageName).add(projectFile);
|
||
}
|
||
}
|
||
return res;
|
||
}
|
||
static void generateForLanguage(String dvm_drv, LanguageName language, Vector<ProjectFile> language_programs, Vector<String> titles, Vector<String> objects, Vector<String> bodies, String flags) {
|
||
if (!language_programs.isEmpty()) {
|
||
String LANG_ = language.toString().toUpperCase() + "_";
|
||
Vector<String> module_objects = new Vector<>();
|
||
String module_body = "";
|
||
int i = 1;
|
||
for (ProjectFile program : language_programs) {
|
||
//--
|
||
String object = Utils.DQuotes(language + "_" + i + ".o");
|
||
module_objects.add(object);
|
||
module_body += object + ":\n" + "\t" + String.join(" ", Utils.MFVar(LANG_ + "COMMAND"), Utils.MFVar(LANG_ + "FLAGS"), program.getStyleOptions(), "-c", program.getQSourceName(), "-o", object + "\n\n");
|
||
++i;
|
||
}
|
||
titles.add(String.join("\n", LANG_ + "COMMAND=" + Utils.DQuotes(dvm_drv) + " " + language.getDVMCompile(), LANG_ + "FLAGS=" + flags, LANG_ + "OBJECTS=" + String.join(" ", module_objects), ""));
|
||
objects.add(Utils.MFVar(LANG_ + "OBJECTS"));
|
||
bodies.add(module_body);
|
||
}
|
||
}
|
||
static String generateMakefile(File test, LanguageName test_language, String dvm_drv, String flags) {
|
||
//----->>
|
||
LinkedHashMap<LanguageName, Vector<ProjectFile>> programs = getTestPrograms(test);
|
||
Vector<String> titles = new Vector<>();
|
||
Vector<String> objects = new Vector<>();
|
||
Vector<String> bodies = new Vector<>();
|
||
String binary = Utils.DQuotes("0");
|
||
//----->>
|
||
for (LanguageName languageName : programs.keySet()) {
|
||
generateForLanguage(dvm_drv, languageName, programs.get(languageName), titles, objects, bodies, flags);
|
||
}
|
||
//----->>
|
||
return String.join("\n", "LINK_COMMAND=" + Utils.DQuotes(dvm_drv) + " " + test_language.getDVMLink(), "LINK_FLAGS=" + flags + "\n", String.join("\n", titles), "all: " + binary, binary + " : " + String.join(" ", objects), "\t" + Utils.MFVar("LINK_COMMAND") + " " + Utils.MFVar("LINK_FLAGS") + " " + String.join(" ", objects) + " -o " + binary, String.join(" ", bodies));
|
||
}
|
||
public void getTasksInfo(List<? extends DVMTask> tasks, String file_name) throws Exception {
|
||
LinkedHashMap<Integer, DVMTask> sorted_tasks = new LinkedHashMap<>();
|
||
for (DVMTask task : tasks)
|
||
sorted_tasks.put(task.id, task);
|
||
//--
|
||
File info_file = Paths.get(packageLocalWorkspace.getAbsolutePath(), "results", file_name).toFile();
|
||
List<String> lines = FileUtils.readLines(info_file, Charset.defaultCharset());
|
||
for (String packed : lines) {
|
||
if (!packed.isEmpty()) {
|
||
String[] data = packed.split(" ");
|
||
int id = Integer.parseInt(data[0]);
|
||
TaskState state = TaskState.valueOf(data[1]);
|
||
double time = Double.parseDouble(data[2]);
|
||
//--
|
||
DVMTask task = sorted_tasks.get(id);
|
||
task.state = state;
|
||
task.Time = state.equals(TaskState.AbortedByTimeout) ? (task.maxtime + 1) : time;
|
||
}
|
||
}
|
||
}
|
||
protected boolean CheckModules() throws Exception {
|
||
RemoteFile modulesDirectory = new RemoteFile(user.workspace, "modules");
|
||
RemoteFile version = new RemoteFile(modulesDirectory, "version.h");
|
||
int current_version = Constants.Nan;
|
||
int actual_version = Constants.planner_version;
|
||
if (user.connection.Exists(version)) {
|
||
try {
|
||
current_version = Integer.parseInt(user.connection.readFromFile(version));
|
||
} catch (Exception ex) {
|
||
ex.printStackTrace();
|
||
}
|
||
}
|
||
if (current_version < actual_version) {
|
||
Print("Закачка кода модулей...");
|
||
for (String resource_name : Constants.resourses_names) {
|
||
Print(resource_name);
|
||
user.connection.putResource(modulesDirectory, resource_name);
|
||
}
|
||
//--
|
||
Print("Сборка модулей...");
|
||
String modules_log = user.connection.compileModules(modulesDirectory);
|
||
if (!modules_log.isEmpty()) {
|
||
testingPackage.description = modules_log;
|
||
testingPackage.state = TasksPackageState.Aborted;
|
||
return false;
|
||
}
|
||
}
|
||
return true;
|
||
}
|
||
//----
|
||
@Override
|
||
protected ServerCode getActivePackagesCode() {
|
||
return ServerCode.GetFirstActiveDVMPackageForMachineURL;
|
||
}
|
||
@Override
|
||
protected ServerCode getCheckIfNeedsKillCode() {
|
||
return ServerCode.DVMPackageNeedsKill;
|
||
}
|
||
@Override
|
||
protected TasksPackageState getStateAfterStart() {
|
||
return TasksPackageState.CompilationWorkspacesCreation;
|
||
}
|
||
@Override
|
||
protected void TestsSynchronize() throws Exception {
|
||
testingPackage.readJson();
|
||
LinkedHashMap<Integer, File> tests = getTestsFromJson();
|
||
//синхронизировать их.
|
||
for (int test_id : tests.keySet()) {
|
||
// Print("testId="+test_id);
|
||
File test = tests.get(test_id);
|
||
RemoteFile test_dst = new RemoteFile(testingPackage.user_workspace + "/projects/" + test_id, true);
|
||
// Print("src="+test.getAbsolutePath());
|
||
// Print("dst="+test_dst.full_name);
|
||
user.connection.MKDIR(test_dst);
|
||
user.connection.SynchronizeSubDirsR(test, test_dst);
|
||
// Print("done");
|
||
}
|
||
}
|
||
@Override
|
||
protected void PackageWorkspaceCreation() throws Exception {
|
||
if (!CheckModules()) {
|
||
return;
|
||
}
|
||
//--
|
||
testingPackage.readJson();
|
||
//--
|
||
LinkedHashMap<Integer, File> tests = getTestsFromJson();
|
||
//создать папку для пакета.
|
||
user.connection.sftpChannel.mkdir(packageRemoteWorkspace.full_name);
|
||
//положить туда запакованные тексты задач.
|
||
Vector<String> compilationLines = new Vector<>();
|
||
Vector<String> runLines = new Vector<>();
|
||
for (DVMCompilationTask compilationTask : testingPackage.package_json.compilationTasks) {
|
||
String makefileText = generateMakefile(tests.get(compilationTask.test_id), compilationTask.language, testingPackage.drv, compilationTask.flags);
|
||
compilationLines.addAll(compilationTask.pack(makefileText));
|
||
for (DVMRunTask runTask : compilationTask.runTasks)
|
||
runLines.addAll(runTask.pack(null));
|
||
}
|
||
RemoteFile compilationPackage = new RemoteFile(packageRemoteWorkspace, "compilationTasks");
|
||
RemoteFile runPackage = new RemoteFile(packageRemoteWorkspace, "runTasks");
|
||
user.connection.writeToFile(String.join("\n", compilationLines) + "\n", compilationPackage);
|
||
user.connection.writeToFile(String.join("\n", runLines) + "\n", runPackage);
|
||
// --
|
||
user.connection.MKDIR(new RemoteFile(packageRemoteWorkspace, "state"));
|
||
}
|
||
@Override
|
||
protected void AnalyseResults() throws Exception {
|
||
testingPackage.readJson();
|
||
Print("analysing results");
|
||
Vector<DVMRunTask> runTasks = new Vector<>();
|
||
for (DVMCompilationTask compilationTask : testingPackage.package_json.compilationTasks)
|
||
runTasks.addAll(compilationTask.runTasks);
|
||
//----
|
||
getTasksInfo(testingPackage.package_json.compilationTasks, "CompilationInfo.txt");
|
||
getTasksInfo(runTasks, "RunningInfo.txt");
|
||
//--
|
||
int ct_count = 0;
|
||
int rt_count = 0;
|
||
//--
|
||
for (DVMCompilationTask compilationTask : testingPackage.package_json.compilationTasks) {
|
||
compilationTask.dvm_package_id = testingPackage.id;
|
||
ct_count++;
|
||
File ct_workspace = Paths.get(packageLocalWorkspace.getAbsolutePath(), "results", String.valueOf(compilationTask.id)).toFile();
|
||
if (ct_workspace.exists()) {
|
||
for (DVMRunTask runTask : compilationTask.runTasks) {
|
||
runTask.dvm_package_id = testingPackage.id;
|
||
rt_count++;
|
||
runTask.compilation_state = compilationTask.state;
|
||
runTask.compilation_time = compilationTask.Time;
|
||
if (compilationTask.state == TaskState.DoneWithErrors) {
|
||
runTask.state = TaskState.Canceled;
|
||
} else {
|
||
File rt_workspace = Paths.get(packageLocalWorkspace.getAbsolutePath(), "results", String.valueOf(runTask.id)).toFile();
|
||
if (rt_workspace.exists() && runTask.state.equals(TaskState.Finished)) {
|
||
//анализ задачи на запуск.
|
||
File outFile = new File(rt_workspace, Constants.out_file);
|
||
File errFile = new File(rt_workspace.getAbsolutePath(), Constants.err_file);
|
||
//--
|
||
String output = FileUtils.readFileToString(outFile);
|
||
String errors = FileUtils.readFileToString(errFile);
|
||
//--
|
||
List<String> output_lines = Arrays.asList(output.split("\n"));
|
||
List<String> errors_lines = Arrays.asList(errors.split("\n"));
|
||
//---
|
||
if (Utils.isCrushed(output_lines, errors_lines)) {
|
||
runTask.state = TaskState.Crushed;
|
||
} else {
|
||
Pair<TaskState, Integer> results = new Pair<>(TaskState.Done, 100);
|
||
switch (runTask.test_type) {
|
||
case Correctness:
|
||
results = Utils.analyzeCorrectness(output_lines);
|
||
break;
|
||
case Performance:
|
||
results = Utils.analyzePerformance(output_lines);
|
||
break;
|
||
default:
|
||
break;
|
||
}
|
||
runTask.state = results.getKey();
|
||
runTask.progress = results.getValue();
|
||
runTask.CleanTime = Utils.parseCleanTime(output);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
testingPackage.progress = 100;
|
||
testingPackage.saveJson(); //запись обновленных результатов пакета в json!
|
||
Print("analysis done, ct_count=" + ct_count + " rt count=" + rt_count);
|
||
}
|
||
@Override
|
||
protected void PackageStart() throws Exception {
|
||
String plannerStartCommand = String.join(" ",
|
||
Utils.DQuotes(getPlanner()),
|
||
Utils.DQuotes(user.workspace),
|
||
Utils.DQuotes(packageRemoteWorkspace.full_name),
|
||
Utils.DQuotes(testingPackage.kernels),
|
||
Utils.DQuotes(testingPackage.drv));
|
||
user.connection.startShellProcess(packageRemoteWorkspace, "planner_output",
|
||
"ulimit -s unlimited", plannerStartCommand);
|
||
//---
|
||
RemoteFile PID = new RemoteFile(packageRemoteWorkspace, "PID");
|
||
while (!user.connection.Exists(PID)) {
|
||
Print("PID not found");
|
||
Utils.sleep(1000);
|
||
}
|
||
testingPackage.PID = user.connection.readFromFile(PID).replace("\n", "").replace("\r", "");
|
||
//---
|
||
System.out.println("PID=" + Utils.Brackets(testingPackage.PID));
|
||
RemoteFile STARTED = new RemoteFile(packageRemoteWorkspace, "STARTED");
|
||
while (!user.connection.Exists(STARTED)) {
|
||
Print("waiting for package start...");
|
||
Utils.sleep(1000);
|
||
}
|
||
}
|
||
@Override
|
||
protected boolean CheckNextState() throws Exception {
|
||
boolean progress_changed = false;
|
||
boolean state_changed = false;
|
||
RemoteFile progress = new RemoteFile(packageRemoteWorkspace, "progress");
|
||
if (user.connection.Exists(progress)) {
|
||
String s = user.connection.readFromFile(progress);
|
||
int current_progress = Integer.parseInt(s);
|
||
if (current_progress != testingPackage.progress) {
|
||
Print("progress changed: " + current_progress);
|
||
testingPackage.progress = current_progress;
|
||
progress_changed = true;
|
||
}
|
||
}
|
||
RemoteFile stateDir = new RemoteFile(packageRemoteWorkspace, "state");
|
||
//состояния пакета могут меняться только по возрастанию. ищем, появилось ли такое.
|
||
Vector<TasksPackageState> higherStates = testingPackage.state.getHigherStates();
|
||
Collections.reverse(higherStates); //берем в обратном порядке, чтобы быстрее найти высшее.
|
||
for (TasksPackageState state : higherStates) {
|
||
RemoteFile file = new RemoteFile(stateDir, state.toString());
|
||
if (user.connection.Exists(file)) {
|
||
Print("found new state: " + file.name);
|
||
testingPackage.state = state;
|
||
state_changed = true;
|
||
break;
|
||
}
|
||
}
|
||
//--
|
||
user.connection.iterations++;
|
||
if (user.connection.iterations == 100) {
|
||
Disconnect();
|
||
}
|
||
//--
|
||
return progress_changed || state_changed;
|
||
}
|
||
@Override
|
||
protected void DownloadResults() throws Exception {
|
||
Utils.CheckDirectory(packageLocalWorkspace);
|
||
RemoteFile remote_results_archive = new RemoteFile(packageRemoteWorkspace, "results.zip");
|
||
File results_archive = new File(packageLocalWorkspace, "results.zip");
|
||
user.connection.performScript(packageRemoteWorkspace, "zip -r " + Utils.DQuotes("results.zip") + " " + Utils.DQuotes("results"));
|
||
//---
|
||
if (user.connection.Exists(remote_results_archive)) {
|
||
user.connection.getSingleFile(remote_results_archive.full_name, results_archive.getAbsolutePath());
|
||
UnzipFolderPass unzipFolderPass = new UnzipFolderPass();
|
||
unzipFolderPass.Do(results_archive.getAbsolutePath(), packageLocalWorkspace.getAbsolutePath(), false);
|
||
}
|
||
//---
|
||
// if (Global.properties.eraseTestingWorkspaces && user.connection.Exists(packageRemoteWorkspace))
|
||
// user.connection.RMDIR(packageRemoteWorkspace.full_name);
|
||
}
|
||
@Override
|
||
protected void MachineConnectionError() {
|
||
Finalize("Количество безуспешных попыток соединения с машиной " + machine.getURL() + " превысило 10");
|
||
}
|
||
@Override
|
||
protected void ServerConnectionError(ServerCode code_in, String logText) throws Exception {
|
||
Finalize("Не удалось выполнить команду " + code_in + " на сервере тестирования\n" + logText);
|
||
}
|
||
@Override
|
||
protected void Kill() throws Exception {
|
||
if (!testingPackage.PID.isEmpty()) {
|
||
user.connection.Command("kill -9 " + testingPackage.PID);
|
||
}
|
||
}
|
||
@Override
|
||
protected void InitSessionCredentials() {
|
||
packageRemoteWorkspace = new RemoteFile(user.workspace + "/tests", String.valueOf(testingPackage.id), true);
|
||
packageLocalWorkspace = new File(Global.DVMPackagesDirectory, String.valueOf(testingPackage.id));
|
||
}
|
||
@Override
|
||
public void perform() throws Exception {
|
||
Print("Проверка сервера...");
|
||
String currentServerName = (String) ServerCommand(ServerCode.GetServerName);
|
||
Print("имя текущего сервера " + Utils.Brackets(currentServerName));
|
||
Print("имя сервера, создавшего нить " + Utils.Brackets(serverName));
|
||
if (!serverName.equals(currentServerName)) {
|
||
Finalize("Несоответствующий сервер");
|
||
}
|
||
Print("Запрос активных пакетов для машины " + Utils.Brackets(machine.getURL()));
|
||
testingPackage = null;
|
||
Vector<DVMPackage> activePackages = (Vector<DVMPackage>) ServerCommand(getActivePackagesCode(), machine.getURL(), null);
|
||
if (activePackages.isEmpty())
|
||
Finalize("Не найдено активных пакетов для машины " + Utils.Brackets(machine.getURL()));
|
||
for (DVMPackage activePackage : activePackages)
|
||
PerformPackage(activePackage);
|
||
}
|
||
}
|