M/VisualSapfor
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Наметил локальный вариант, но пока не уверен что он приоритетен..

Browse Source
This commit is contained in:
02090095
2024-04-27 18:44:36 +03:00
parent c84a648bb6
commit 002a8517e0
14 changed files with 421 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.idea/workspace.xml generated
View File

@@ -7,10 +7,18 @@
</component>
<component name="ChangeListManager">
<list default="true" id="e42177c3-2328-4b27-8a01-35779b2beb99" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/src/TestingSystem/DVM/DVMTestingPlanner.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/properties" beforeDir="false" afterPath="$PROJECT_DIR$/properties" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/Common/Current.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Common/Current.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/Common/Global.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Common/Global.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/Repository/Component/Visualiser.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Repository/Component/Visualiser.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/Repository/RepositoryClient.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Repository/RepositoryClient.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/Repository/RepositoryServer.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Repository/RepositoryServer.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/Common/TestingServer.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/Common/TestingServer.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/Common/MachineProcess/MachineProcess.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/Common/MachineProcess/MachineProcess.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/Common/TestingPlanner.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/Common/TestingPlanner.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/DVM/LocalMachineQueueSupervisor.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/DVM/LocalDVMTestingPlanner.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/DVM/MachineQueueSupervisor.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/DVM/RemoteDVMTestingPlanner.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/DVM/UserConnection.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/DVM/UserConnection.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/SSH/RepositoryServerSSHPass.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/SSH/RepositoryServerSSHPass.java" afterDir="false" />
</list>

3
src/Common/Current.java
View File

@@ -481,6 +481,7 @@ public enum Current {
Server,
Testing,
Package,
MachineQueue
MachineQueue,
LocalMachineQueue
}
}

14
src/Common/Global.java
View File

@@ -13,7 +13,8 @@ import Repository.Component.Sapfor.Sapfor_F;
import Repository.Component.Sapfor.TransformationPermission;
import Repository.Server.ComponentsServer;
import TestingSystem.Common.TestingServer;
import TestingSystem.DVM.MachineQueueSupervisor;
import TestingSystem.DVM.LocalDVMTestingPlanner;
import TestingSystem.DVM.RemoteDVMTestingPlanner;
import TestingSystem.SAPFOR.PackageModeSupervisor;
import Visual_DVM_2021.Passes.PassCode_2021;
import Visual_DVM_2021.Passes.Pass_2021;
@@ -388,7 +389,13 @@ public class Global {
planner.Start();
}
public static void MachineQueueMode(String... args) throws Exception {
MachineQueueSupervisor supervisor = new MachineQueueSupervisor(args);
RemoteDVMTestingPlanner supervisor = new RemoteDVMTestingPlanner(args);
while (true) {
supervisor.Perform();
}
}
public static void LocalMachineQueueMode(String... args) throws Exception {
LocalDVMTestingPlanner supervisor = new LocalDVMTestingPlanner(args);
while (true) {
supervisor.Perform();
}
@@ -419,6 +426,9 @@ public class Global {
case MachineQueue:
MachineQueueMode(args);
break;
case LocalMachineQueue:
LocalMachineQueueMode(args);
break;
case Undefined:
break;
}

2
src/Repository/Component/Visualiser.java
View File

@@ -62,7 +62,7 @@ public class Visualiser extends Component {
//http://www.seostella.com/ru/article/2012/02/05/formatirovanie-daty-v-java.html
@Override
public void GetVersionInfo() {
version = 1097;
version = 1098;
String pattern = "MMM dd yyyy HH:mm:ss";
DateFormat df = new SimpleDateFormat(pattern, Locale.ENGLISH);
date_text = df.format(getClassBuildTime());

3
src/Repository/RepositoryClient.java
View File

@@ -1,10 +1,13 @@
package Repository;
import Common.Constants;
import Common.Utils.Utils;
import Repository.Server.ServerCode;
import Repository.Server.ServerExchangeUnit_2021;
import Visual_DVM_2021.Passes.PassException;
import Visual_DVM_2021.Passes.Server.TestingSystemPass;
import org.apache.commons.io.FileUtils;
import java.io.File;
import java.io.FileWriter;
import java.io.Serializable;
import java.util.Date;

6
src/Repository/RepositoryServer.java
View File

@@ -9,6 +9,7 @@ import Repository.Server.DiagnosticSignalHandler;
import Repository.Server.ServerCode;
import Repository.Server.ServerExchangeUnit_2021;
import javafx.util.Pair;
import org.apache.commons.io.FileUtils;
import sun.misc.SignalHandler;
import javax.activation.DataHandler;
@@ -177,7 +178,12 @@ public abstract class RepositoryServer<D extends Database> {
public void StartAction() throws Exception {
}
public void Start() throws Exception {
/*
File started = new File(Constants.STARTED);
if (started.exists())
FileUtils.forceDelete(started);
Utils.createEmptyFile(Constants.STARTED);
*/
DiagnosticSignalHandler.install("TERM", signalHandler);
DiagnosticSignalHandler.install("INT", signalHandler);
DiagnosticSignalHandler.install("ABRT", signalHandler);

21
src/TestingSystem/Common/MachineProcess/MachineProcess.java
View File

@@ -9,6 +9,7 @@ import TestingSystem.DVM.DVMPackage.DVMPackage;
import org.apache.commons.io.FileUtils;
import java.io.File;
import java.net.InetAddress;
import java.util.Vector;
public class MachineProcess extends DBObject {
public String id = "";
@@ -32,7 +33,7 @@ public class MachineProcess extends DBObject {
userWorkspace = p.user_workspace;
testingSystemRoot = Global.Home;
serverName = Global.testingServer.name;
id = Utils.getDateName(machineAddress+"_"+machinePort+"_"+userName);
id = Utils.getDateName(machineAddress + "_" + machinePort + "_" + userName);
}
@Override
public Object getPK() {
@@ -76,6 +77,19 @@ public class MachineProcess extends DBObject {
File started = getStartedFile();
return started.exists();
}
public boolean isLocal() {
boolean local = false;
try {
InetAddress address = InetAddress.getByName(machineAddress);
InetAddress localAddress = InetAddress.getByName(Global.properties.ServerAddress);
System.out.println("machine ip=" + Utils.Brackets(address.getHostAddress()));
System.out.println("server ip=" + Utils.Brackets(localAddress.getHostAddress()));
local = localAddress.getHostAddress().equals(address.getHostAddress());
} catch (Exception ex) {
Global.Log.PrintException(ex);
}
return local;
}
//--
public void Start() {
try {
@@ -87,7 +101,8 @@ public class MachineProcess extends DBObject {
FileUtils.copyFile(src, supervisor);
//создание настроек
GlobalProperties properties = new GlobalProperties(Global.properties);
properties.Mode = Current.Mode.MachineQueue;
properties.Mode = //isLocal()?Current.Mode.LocalMachineQueue:
Current.Mode.MachineQueue;
Utils.jsonToFile(properties, new File(workspace, "properties"));
Vector<String> args = new Vector<>();
args.add(Utils.DQuotes(machineAddress));
@@ -101,7 +116,7 @@ public class MachineProcess extends DBObject {
Utils.startScript(workspace, workspace,
"start",
"java -jar VisualSapfor.jar " +
String.join(" ", args)+" 1>out.txt 2>err.txt");
String.join(" ", args) + " 1>out.txt 2>err.txt");
//---
} catch (Exception ex) {
ex.printStackTrace();

89
src/TestingSystem/Common/TestingPlanner.java
View File

@@ -1,19 +1,32 @@
package TestingSystem.Common;
import Common.Constants;
import Common.Current;
import Common.Global;
import Common.Utils.Utils;
import GlobalData.Machine.Machine;
import GlobalData.Machine.MachineType;
import GlobalData.User.User;
import Repository.EmailMessage;
import Repository.Server.ServerCode;
import TestingSystem.Common.TestingPackage.TestingPackage;
import TestingSystem.Common.TestingPackageToKill.TestingPackageToKill;
import Repository.RepositoryClient;
import Visual_DVM_2021.Passes.PassException;
import Visual_DVM_2021.UI.Interface.Loggable;
import javafx.util.Pair;
import org.apache.commons.io.FileUtils;
import java.io.File;
import java.util.Date;
import java.util.Vector;
public abstract class TestingPlanner<P extends TestingPackage> extends RepositoryClient {
protected P testingPackage;
//---
protected File packageLocalWorkspace = null;
protected String serverName = "";
protected File supervisorHome = null;
protected Machine machine = null;
protected User user = null;
//----
void UpdatePackageState(TasksPackageState state_in) throws Exception {
testingPackage.state = state_in;
testingPackage.ChangeDate = new Date().getTime();
@@ -60,7 +73,6 @@ public abstract class TestingPlanner<P extends TestingPackage> extends Repositor
}
protected void MachineConnectionError() {
}
// ---
protected void PerformPackage(TestingPackage package_in) throws Exception {
testingPackage = (P) package_in;
//--
@@ -140,4 +152,77 @@ public abstract class TestingPlanner<P extends TestingPackage> extends Repositor
for (P activePackage : activePackages)
PerformPackage(activePackage);
}
protected void Finalize(String reason) {
Print(reason);
File stateFile = new File(supervisorHome, Constants.ABORTED);
try {
FileUtils.writeStringToFile(stateFile, reason);
} catch (Exception ex) {
ex.printStackTrace();
}
System.exit(0);
}
//---------------------------------------------
public String getPlanner() {
return String.join("/", user.workspace, "modules", "planner");
}
//---
public TestingPlanner(){}
public TestingPlanner(String... args) {
Global.isWindows = System.getProperty("os.name").startsWith("Windows");
//---
String machineAddress = args[0];
int machinePort = Integer.parseInt(args[1]);
String userName = args[2];
String userPassword = args[3];
String userWorkspace = args[4];
String testingSystemRoot = args[5];
serverName = args[6];
supervisorHome = new File(Global.Home); //при инициализации это текущая папка.
//---
Global.Log = new Loggable() {
@Override
public String getLogHomePath() {
return supervisorHome.getAbsolutePath();
}
@Override
public String getLogName() {
return Current.mode.toString();
}
};
Global.Log.ClearLog();
//--
Global.Home = testingSystemRoot;
Global.CheckTestingSystemDirectories();
System.out.println(Global.TestsDirectory.getAbsolutePath());
//---
machine = new Machine(machineAddress, machineAddress, machinePort, MachineType.Server);
user = new User(userName, userPassword, userWorkspace);
//---
Print("machineAddress=" + Utils.Brackets(machineAddress));
Print("machinePort=" + Utils.Brackets(String.valueOf(machinePort)));
Print("userName=" + Utils.Brackets(userName));
Print("userPassword=" + Utils.Brackets(userPassword));
Print("userWorkspace=" + Utils.Brackets(userWorkspace));
Print("root=" + Utils.Brackets(Global.Home));
Print("serverName=" + serverName);
Print("=====");
//----
Utils.createEmptyFile(Constants.STARTED);
}
/*
void CheckLocal() {
local = false;
try {
InetAddress address = InetAddress.getByName(machine.address);
InetAddress localAddress = InetAddress.getByName("alex-freenas.ddns.net");
Print("machine ip=" + Utils.Brackets(address.getHostAddress()));
Print("server ip=" + Utils.Brackets(localAddress.getHostAddress()));
local = localAddress.getHostAddress().equals(address.getHostAddress());
//todo в этом случае отдельный режим без ssh
} catch (Exception ex) {
Global.Log.PrintException(ex);
}
}
*/
}

154
src/TestingSystem/DVM/DVMTestingPlanner.java Normal file
View File

@@ -0,0 +1,154 @@
package TestingSystem.DVM;
import Common.Constants;
import Common.Global;
import Common.Utils.Utils;
import GlobalData.RemoteFile.RemoteFile;
import GlobalData.Tasks.TaskState;
import ProjectData.Files.ProjectFile;
import ProjectData.LanguageName;
import Repository.Server.ServerCode;
import TestingSystem.Common.TasksPackageState;
import TestingSystem.Common.TestingPlanner;
import TestingSystem.DVM.DVMPackage.DVMPackage;
import TestingSystem.DVM.DVMTasks.DVMCompilationTask;
import TestingSystem.DVM.DVMTasks.DVMTask;
import org.apache.commons.io.FileUtils;
import java.io.File;
import java.io.FileFilter;
import java.nio.charset.Charset;
import java.nio.file.Paths;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Vector;
public abstract class DVMTestingPlanner extends TestingPlanner<DVMPackage> {
public DVMTestingPlanner(String[] args) {
super(args);
}
LinkedHashMap<Integer, File> getTestsFromJson() {
LinkedHashMap<Integer, File> res = new LinkedHashMap<>();
for (DVMCompilationTask task : testingPackage.package_json.compilationTasks) {
if (!res.containsKey(task.test_id)) {
res.put(task.test_id, Paths.get(Global.TestsDirectory.getAbsolutePath(), String.valueOf(task.test_id)).toFile());
}
}
return res;
}
static LinkedHashMap<LanguageName, Vector<ProjectFile>> getTestPrograms(File test) {
LinkedHashMap<LanguageName, Vector<ProjectFile>> res = new LinkedHashMap<>();
//--
res.put(LanguageName.fortran, new Vector<>());
res.put(LanguageName.c, new Vector<>());
res.put(LanguageName.cpp, new Vector<>());
//--
File[] files = test.listFiles(new FileFilter() {
@Override
public boolean accept(File pathname) {
return pathname.isFile();
}
});
if (files != null) {
for (File file : files) {
ProjectFile projectFile = new ProjectFile(new File(file.getName()));
if (projectFile.isNotExcludedProgram()) res.get(projectFile.languageName).add(projectFile);
}
}
return res;
}
static void generateForLanguage(String dvm_drv, LanguageName language, Vector<ProjectFile> language_programs, Vector<String> titles, Vector<String> objects, Vector<String> bodies, String flags) {
if (!language_programs.isEmpty()) {
String LANG_ = language.toString().toUpperCase() + "_";
Vector<String> module_objects = new Vector<>();
String module_body = "";
int i = 1;
for (ProjectFile program : language_programs) {
//--
String object = Utils.DQuotes(language + "_" + i + ".o");
module_objects.add(object);
module_body += object + ":\n" + "\t" + String.join(" ", Utils.MFVar(LANG_ + "COMMAND"), Utils.MFVar(LANG_ + "FLAGS"), program.getStyleOptions(), "-c", program.getQSourceName(), "-o", object + "\n\n");
++i;
}
titles.add(String.join("\n", LANG_ + "COMMAND=" + Utils.DQuotes(dvm_drv) + " " + language.getDVMCompile(), LANG_ + "FLAGS=" + flags, LANG_ + "OBJECTS=" + String.join(" ", module_objects), ""));
objects.add(Utils.MFVar(LANG_ + "OBJECTS"));
bodies.add(module_body);
}
}
static String generateMakefile(File test, LanguageName test_language, String dvm_drv, String flags) {
//----->>
LinkedHashMap<LanguageName, Vector<ProjectFile>> programs = getTestPrograms(test);
Vector<String> titles = new Vector<>();
Vector<String> objects = new Vector<>();
Vector<String> bodies = new Vector<>();
String binary = Utils.DQuotes("0");
//----->>
for (LanguageName languageName : programs.keySet()) {
generateForLanguage(dvm_drv, languageName, programs.get(languageName), titles, objects, bodies, flags);
}
//----->>
return String.join("\n", "LINK_COMMAND=" + Utils.DQuotes(dvm_drv) + " " + test_language.getDVMLink(), "LINK_FLAGS=" + flags + "\n", String.join("\n", titles), "all: " + binary, binary + " : " + String.join(" ", objects), "\t" + Utils.MFVar("LINK_COMMAND") + " " + Utils.MFVar("LINK_FLAGS") + " " + String.join(" ", objects) + " -o " + binary, String.join(" ", bodies));
}
public void getTasksInfo(List<? extends DVMTask> tasks, String file_name) throws Exception {
LinkedHashMap<Integer, DVMTask> sorted_tasks = new LinkedHashMap<>();
for (DVMTask task : tasks)
sorted_tasks.put(task.id, task);
//--
File info_file = Paths.get(packageLocalWorkspace.getAbsolutePath(), "results", file_name).toFile();
List<String> lines = FileUtils.readLines(info_file, Charset.defaultCharset());
for (String packed : lines) {
if (!packed.isEmpty()) {
String[] data = packed.split(" ");
int id = Integer.parseInt(data[0]);
TaskState state = TaskState.valueOf(data[1]);
double time = Double.parseDouble(data[2]);
//--
DVMTask task = sorted_tasks.get(id);
task.state = state;
task.Time = state.equals(TaskState.AbortedByTimeout) ? (task.maxtime + 1) : time;
}
}
}
@Override
protected void Print(String message) {
try {
if (isPrintOn()) {
Global.Log.Print(message);
}
} catch (Exception ex) {
ex.printStackTrace();
}
}
@Override
protected ServerCode getActivePackagesCode() {
return ServerCode.GetFirstActiveDVMPackageForMachineURL;
}
@Override
protected ServerCode getCheckIfNeedsKillCode() {
return ServerCode.DVMPackageNeedsKill;
}
@Override
protected TasksPackageState getStateAfterStart() {
return TasksPackageState.CompilationWorkspacesCreation;
}
@Override
protected void ServerConnectionError(ServerCode code_in, String logText) throws Exception {
Finalize("Не удалось выполнить команду " + code_in + " на сервере тестирования\n" + logText);
}
protected abstract boolean CheckModules() throws Exception;
@Override
public void perform() throws Exception {
Print("Проверка сервера...");
String currentServerName = (String) ServerCommand(ServerCode.GetServerName);
Print("имя текущего сервера " + Utils.Brackets(currentServerName));
Print("имя сервера, создавшего нить " + Utils.Brackets(serverName));
if (!serverName.equals(currentServerName)) {
Finalize("Несоответствующий сервер");
}
Print("Запрос активных пакетов для машины " + Utils.Brackets(machine.getURL()));
testingPackage = null;
Vector<DVMPackage> activePackages = (Vector<DVMPackage>) ServerCommand(getActivePackagesCode(), machine.getURL(), null);
if (activePackages.isEmpty())
Finalize("Не найдено активных пакетов для машины " + Utils.Brackets(machine.getURL()));
for (DVMPackage activePackage : activePackages)
PerformPackage(activePackage);
}
}

92
src/TestingSystem/DVM/LocalDVMTestingPlanner.java Normal file
View File

@@ -0,0 +1,92 @@
package TestingSystem.DVM;
import Common.Constants;
import Common.Global;
import Common.Utils.Utils;
import GlobalData.RemoteFile.RemoteFile;
import Repository.Server.ServerCode;
import TestingSystem.Common.TasksPackageState;
import TestingSystem.Common.TestingPlanner;
import TestingSystem.DVM.DVMPackage.DVMPackage;
import org.apache.commons.io.FileUtils;
import java.io.File;
import java.nio.file.Paths;
import java.util.LinkedHashMap;
import java.util.Vector;
public class LocalDVMTestingPlanner extends DVMTestingPlanner {
public LocalDVMTestingPlanner(String[] args) {
super(args);
}
@Override
protected boolean CheckModules() throws Exception {
/*
File modulesDirectory = Paths.get(user.workspace, "modules").toFile();
File version = new File(modulesDirectory,"version.h");
int current_version = Constants.Nan;
int actual_version = Constants.planner_version;
///--
if (version.exists()) {
try {
current_version = Integer.parseInt(FileUtils.readFileToString(version));
System.out.println("current version ="+current_version);
} catch (Exception ex) {
ex.printStackTrace();
}
}
else {
System.out.println("version not exists");
}
if (current_version < actual_version) {
Print("Закачка кода модулей...");
for (String resource_name : Constants.resourses_names) {
Print(resource_name);
File src = Utils.CreateTempResourceFile(resource_name);
File dst = new File(modulesDirectory, resource_name);
FileUtils.copyFile(src, dst);
}
//--
Print("Сборка модулей...");
String modules_log = user.connection.compileModules(modulesDirectory);
if (!modules_log.isEmpty()) {
testingPackage.description = modules_log;
testingPackage.state = TasksPackageState.Aborted;
return false;
}
}
*/
return true;
}
@Override
protected void TestsSynchronize() throws Exception {
testingPackage.readJson();
LinkedHashMap<Integer, File> tests = getTestsFromJson();
//синхронизировать их.
for (int test_id : tests.keySet()) {
File test = tests.get(test_id);
File testDst = Paths.get(testingPackage.user_workspace,"projects", String.valueOf(test_id)).toFile();
Print(testDst.getAbsolutePath());
FileUtils.copyDirectory(test,testDst);
}
Finalize("+");
}
@Override
protected void PackageWorkspaceCreation() throws Exception {
}
@Override
protected void AnalyseResults() throws Exception {
}
@Override
protected void PackageStart() throws Exception {
}
@Override
protected boolean CheckNextState() throws Exception {
return false;
}
@Override
protected void DownloadResults() throws Exception {
}
@Override
protected void Kill() throws Exception {
}
}

41
src/TestingSystem/DVM/LocalMachineQueueSupervisor.java
View File

@@ -1,41 +0,0 @@
package TestingSystem.DVM;
import Repository.Server.ServerCode;
import TestingSystem.Common.TasksPackageState;
import TestingSystem.Common.TestingPlanner;
import TestingSystem.DVM.DVMPackage.DVMPackage;
public class LocalMachineQueueSupervisor extends TestingPlanner<DVMPackage> {
@Override
protected ServerCode getActivePackagesCode() {
return ServerCode.GetFirstActiveDVMPackageForMachineURL;
}
@Override
protected ServerCode getCheckIfNeedsKillCode() {
return ServerCode.DVMPackageNeedsKill;
}
@Override
protected TasksPackageState getStateAfterStart() {
return TasksPackageState.CompilationWorkspacesCreation;
}
@Override
protected void TestsSynchronize() throws Exception {
}
@Override
protected void PackageWorkspaceCreation() throws Exception {
}
@Override
protected void AnalyseResults() throws Exception {
}
@Override
protected void PackageStart() throws Exception {
}
@Override
protected boolean CheckNextState() throws Exception {
return false;
}
@Override
protected void DownloadResults() throws Exception {
}
@Override
protected void Kill() throws Exception {
}
}

267
src/TestingSystem/DVM/MachineQueueSupervisor.java → src/TestingSystem/DVM/RemoteDVMTestingPlanner.java
View File

@@ -28,74 +28,10 @@ import java.net.InetAddress;
import java.nio.charset.Charset;
import java.nio.file.Paths;
import java.util.*;
public class MachineQueueSupervisor extends TestingPlanner<DVMPackage> {
File supervisorHome = null;
Machine machine = null;
User user = null;
boolean local;
public class RemoteDVMTestingPlanner extends DVMTestingPlanner {
RemoteFile packageRemoteWorkspace = null;
File packageLocalWorkspace = null;
String serverName = "";
//----
public MachineQueueSupervisor(String... args) {
Global.isWindows = System.getProperty("os.name").startsWith("Windows");
//---
String machineAddress = args[0];
int machinePort = Integer.parseInt(args[1]);
String userName = args[2];
String userPassword = args[3];
String userWorkspace = args[4];
String testingSystemRoot = args[5];
serverName = args[6];
supervisorHome = new File(Global.Home); //при инициализации это текущая папка.
//---
Global.Log = new Loggable() {
@Override
public String getLogHomePath() {
return supervisorHome.getAbsolutePath();
}
@Override
public String getLogName() {
return Current.mode.toString();
}
};
Global.Log.ClearLog();
//--
Global.Home = testingSystemRoot;
Global.CheckTestingSystemDirectories();
System.out.println(Global.TestsDirectory.getAbsolutePath());
//---
machine = new Machine(machineAddress, machineAddress, machinePort, MachineType.Server);
user = new User(userName, userPassword, userWorkspace);
CheckLocal();
//---
Print("machineAddress=" + Utils.Brackets(machineAddress));
Print("machinePort=" + Utils.Brackets(String.valueOf(machinePort)));
Print("userName=" + Utils.Brackets(userName));
Print("userPassword=" + Utils.Brackets(userPassword));
Print("userWorkspace=" + Utils.Brackets(userWorkspace));
Print("root=" + Utils.Brackets(Global.Home));
Print("local=" + local);
Print("serverName=" + serverName);
Print("=====");
//----
Utils.createEmptyFile(Constants.STARTED);
}
void CheckLocal() {
local = false;
try {
InetAddress address = InetAddress.getByName(machine.address);
InetAddress localAddress = InetAddress.getByName("alex-freenas.ddns.net");
Print("machine ip=" + Utils.Brackets(address.getHostAddress()));
Print("server ip=" + Utils.Brackets(localAddress.getHostAddress()));
local = localAddress.getHostAddress().equals(address.getHostAddress());
//todo в этом случае отдельный режим без ssh
} catch (Exception ex) {
Global.Log.PrintException(ex);
}
}
public String getPlanner() {
return String.join("/", user.workspace, "modules", "planner");
public RemoteDVMTestingPlanner(String... args) {
super(args);
}
@Override
protected boolean Connect() {
@@ -119,167 +55,18 @@ public class MachineQueueSupervisor extends TestingPlanner<DVMPackage> {
Print("Соединение c " + machine.getURL() + " " + user.login + " сброшено.");
}
}
@Override
protected void Print(String message) {
try {
if (isPrintOn()) {
// System.out.println(message);
Global.Log.Print(message);
}
} catch (Exception ex) {
ex.printStackTrace();
}
}
void Finalize(String reason) {
Print(reason);
File stateFile = new File(supervisorHome, Constants.ABORTED);
try {
FileUtils.writeStringToFile(stateFile, reason);
} catch (Exception ex) {
ex.printStackTrace();
}
System.exit(0);
}
//--
//Получить ид тестов и их папки на сервере.
LinkedHashMap<Integer, File> getTestsFromJson() {
LinkedHashMap<Integer, File> res = new LinkedHashMap<>();
for (DVMCompilationTask task : testingPackage.package_json.compilationTasks) {
if (!res.containsKey(task.test_id)) {
res.put(task.test_id, Paths.get(Global.TestsDirectory.getAbsolutePath(), String.valueOf(task.test_id)).toFile());
}
}
return res;
}
static LinkedHashMap<LanguageName, Vector<ProjectFile>> getTestPrograms(File test) {
LinkedHashMap<LanguageName, Vector<ProjectFile>> res = new LinkedHashMap<>();
//--
res.put(LanguageName.fortran, new Vector<>());
res.put(LanguageName.c, new Vector<>());
res.put(LanguageName.cpp, new Vector<>());
//--
File[] files = test.listFiles(new FileFilter() {
@Override
public boolean accept(File pathname) {
return pathname.isFile();
}
});
if (files != null) {
for (File file : files) {
ProjectFile projectFile = new ProjectFile(new File(file.getName()));
if (projectFile.isNotExcludedProgram()) res.get(projectFile.languageName).add(projectFile);
}
}
return res;
}
static void generateForLanguage(String dvm_drv, LanguageName language, Vector<ProjectFile> language_programs, Vector<String> titles, Vector<String> objects, Vector<String> bodies, String flags) {
if (!language_programs.isEmpty()) {
String LANG_ = language.toString().toUpperCase() + "_";
Vector<String> module_objects = new Vector<>();
String module_body = "";
int i = 1;
for (ProjectFile program : language_programs) {
//--
String object = Utils.DQuotes(language + "_" + i + ".o");
module_objects.add(object);
module_body += object + ":\n" + "\t" + String.join(" ", Utils.MFVar(LANG_ + "COMMAND"), Utils.MFVar(LANG_ + "FLAGS"), program.getStyleOptions(), "-c", program.getQSourceName(), "-o", object + "\n\n");
++i;
}
titles.add(String.join("\n", LANG_ + "COMMAND=" + Utils.DQuotes(dvm_drv) + " " + language.getDVMCompile(), LANG_ + "FLAGS=" + flags, LANG_ + "OBJECTS=" + String.join(" ", module_objects), ""));
objects.add(Utils.MFVar(LANG_ + "OBJECTS"));
bodies.add(module_body);
}
}
static String generateMakefile(File test, LanguageName test_language, String dvm_drv, String flags) {
//----->>
LinkedHashMap<LanguageName, Vector<ProjectFile>> programs = getTestPrograms(test);
Vector<String> titles = new Vector<>();
Vector<String> objects = new Vector<>();
Vector<String> bodies = new Vector<>();
String binary = Utils.DQuotes("0");
//----->>
for (LanguageName languageName : programs.keySet()) {
generateForLanguage(dvm_drv, languageName, programs.get(languageName), titles, objects, bodies, flags);
}
//----->>
return String.join("\n", "LINK_COMMAND=" + Utils.DQuotes(dvm_drv) + " " + test_language.getDVMLink(), "LINK_FLAGS=" + flags + "\n", String.join("\n", titles), "all: " + binary, binary + " : " + String.join(" ", objects), "\t" + Utils.MFVar("LINK_COMMAND") + " " + Utils.MFVar("LINK_FLAGS") + " " + String.join(" ", objects) + " -o " + binary, String.join(" ", bodies));
}
public void getTasksInfo(List<? extends DVMTask> tasks, String file_name) throws Exception {
LinkedHashMap<Integer, DVMTask> sorted_tasks = new LinkedHashMap<>();
for (DVMTask task : tasks)
sorted_tasks.put(task.id, task);
//--
File info_file = Paths.get(packageLocalWorkspace.getAbsolutePath(), "results", file_name).toFile();
List<String> lines = FileUtils.readLines(info_file, Charset.defaultCharset());
for (String packed : lines) {
if (!packed.isEmpty()) {
String[] data = packed.split(" ");
int id = Integer.parseInt(data[0]);
TaskState state = TaskState.valueOf(data[1]);
double time = Double.parseDouble(data[2]);
//--
DVMTask task = sorted_tasks.get(id);
task.state = state;
task.Time = state.equals(TaskState.AbortedByTimeout) ? (task.maxtime + 1) : time;
}
}
}
protected boolean CheckModules() throws Exception {
RemoteFile modulesDirectory = new RemoteFile(user.workspace, "modules");
RemoteFile version = new RemoteFile(modulesDirectory, "version.h");
int current_version = Constants.Nan;
int actual_version = Constants.planner_version;
if (user.connection.Exists(version)) {
try {
current_version = Integer.parseInt(user.connection.readFromFile(version));
} catch (Exception ex) {
ex.printStackTrace();
}
}
if (current_version < actual_version) {
Print("Закачка кода модулей...");
for (String resource_name : Constants.resourses_names) {
Print(resource_name);
user.connection.putResource(modulesDirectory, resource_name);
}
//--
Print("Сборка модулей...");
String modules_log = user.connection.compileModules(modulesDirectory);
if (!modules_log.isEmpty()) {
testingPackage.description = modules_log;
testingPackage.state = TasksPackageState.Aborted;
return false;
}
}
return true;
}
//----
@Override
protected ServerCode getActivePackagesCode() {
return ServerCode.GetFirstActiveDVMPackageForMachineURL;
}
@Override
protected ServerCode getCheckIfNeedsKillCode() {
return ServerCode.DVMPackageNeedsKill;
}
@Override
protected TasksPackageState getStateAfterStart() {
return TasksPackageState.CompilationWorkspacesCreation;
}
@Override
protected void TestsSynchronize() throws Exception {
testingPackage.readJson();
LinkedHashMap<Integer, File> tests = getTestsFromJson();
//синхронизировать их.
for (int test_id : tests.keySet()) {
// Print("testId="+test_id);
File test = tests.get(test_id);
RemoteFile test_dst = new RemoteFile(testingPackage.user_workspace + "/projects/" + test_id, true);
// Print("src="+test.getAbsolutePath());
// Print("dst="+test_dst.full_name);
user.connection.MKDIR(test_dst);
user.connection.SynchronizeSubDirsR(test, test_dst);
// Print("done");
}
}
@Override
@@ -455,10 +242,7 @@ public class MachineQueueSupervisor extends TestingPlanner<DVMPackage> {
protected void MachineConnectionError() {
Finalize("Количество безуспешных попыток соединения с машиной " + machine.getURL() + " превысило 10");
}
@Override
protected void ServerConnectionError(ServerCode code_in, String logText) throws Exception {
Finalize("Не удалось выполнить команду " + code_in + " на сервере тестирования\n" + logText);
}
@Override
protected void Kill() throws Exception {
if (!testingPackage.PID.isEmpty()) {
@@ -471,20 +255,33 @@ public class MachineQueueSupervisor extends TestingPlanner<DVMPackage> {
packageLocalWorkspace = new File(Global.DVMPackagesDirectory, String.valueOf(testingPackage.id));
}
@Override
public void perform() throws Exception {
Print("Проверка сервера...");
String currentServerName = (String) ServerCommand(ServerCode.GetServerName);
Print("имя текущего сервера " + Utils.Brackets(currentServerName));
Print("имя сервера, создавшего нить " + Utils.Brackets(serverName));
if (!serverName.equals(currentServerName)) {
Finalize("Несоответствующий сервер");
protected boolean CheckModules() throws Exception {
RemoteFile modulesDirectory = new RemoteFile(user.workspace, "modules");
RemoteFile version = new RemoteFile(modulesDirectory, "version.h");
int current_version = Constants.Nan;
int actual_version = Constants.planner_version;
if (user.connection.Exists(version)) {
try {
current_version = Integer.parseInt(user.connection.readFromFile(version));
} catch (Exception ex) {
ex.printStackTrace();
}
}
Print("Запрос активных пакетов для машины " + Utils.Brackets(machine.getURL()));
testingPackage = null;
Vector<DVMPackage> activePackages = (Vector<DVMPackage>) ServerCommand(getActivePackagesCode(), machine.getURL(), null);
if (activePackages.isEmpty())
Finalize("Не найдено активных пакетов для машины " + Utils.Brackets(machine.getURL()));
for (DVMPackage activePackage : activePackages)
PerformPackage(activePackage);
if (current_version < actual_version) {
Print("Закачка кода модулей...");
for (String resource_name : Constants.resourses_names) {
Print(resource_name);
user.connection.putResource(modulesDirectory, resource_name);
}
//--
Print("Сборка модулей...");
String modules_log = user.connection.compileModules(modulesDirectory);
if (!modules_log.isEmpty()) {
testingPackage.description = modules_log;
testingPackage.state = TasksPackageState.Aborted;
return false;
}
}
return true;
}
}

6
src/TestingSystem/DVM/UserConnection.java
View File

@@ -468,11 +468,10 @@ public class UserConnection {
}
public void waitForFileCreation(RemoteFile file) throws Exception {
while (!Exists(file)){
System.out.println(file.name +" NOT FOUND");
System.out.println(file.full_name +" NOT FOUND");
Utils.sleep(1000);
}
}
//возможно в дальнейшем обобщить чтобы вместо PID был файл вывода с задаваемым именем?
public String startShellProcess(RemoteFile directory, String outFileName, String... commands) throws Exception {
Vector<String> commands_ = new Vector<>();
commands_.add("cd " + Utils.DQuotes(directory.full_name));
@@ -490,8 +489,11 @@ public class UserConnection {
String start_command = Utils.DQuotes(script_file.full_name);
//--
RemoteFile outFile = new RemoteFile(directory, outFileName);
if (Exists(outFile))
sftpChannel.rm(outFile.full_name);
ShellConnect();
pin.write(("nohup " + start_command + " &\r\n").getBytes());
waitForFileCreation(outFile);
ShellDisconnect();
return readFromFile(outFile).replace("\n","").replace("\r","");

5
src/Visual_DVM_2021/Passes/SSH/RepositoryServerSSHPass.java
View File

@@ -43,16 +43,13 @@ public abstract class RepositoryServerSSHPass extends ConnectionPass {
protected void StartServer() throws Exception {
user.connection.startShellProcess(serverHome,"server_out.txt", "java -jar " + getServerFileName());
//--
RemoteFile STARTED = new RemoteFile(serverHome, Constants.STARTED);
user.connection.waitForFileCreation(STARTED);
//---
RemoteFile StartLog = new RemoteFile(serverHome, Constants.STARTED +
"_by_" +
Current.getAccount().email +
"_" +
new Date().toString()
);
user.connection.sftpChannel.rename(STARTED.full_name, StartLog.full_name);
user.connection.writeToFile("", StartLog);
//--
RemoteFile[] files = new RemoteFile[]{
new RemoteFile(serverHome, Constants.script),