M/VisualSapfor
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Оптимизация команды Exists для SSH

Browse Source
This commit is contained in:
02090095
2023-12-04 14:42:36 +03:00
parent 1f8ebdc9a2
commit 163552d74f
13 changed files with 93 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.idea/workspace.xml generated
View File

@@ -8,12 +8,14 @@
<component name="ChangeListManager">
<list default="true" id="e42177c3-2328-4b27-8a01-35779b2beb99" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/GlobalData/Tasks/TaskState.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/GlobalData/Tasks/TaskState.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/Common/TestingServer.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/Common/TestingServer.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/properties" beforeDir="false" afterPath="$PROJECT_DIR$/properties" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/GlobalData/Tasks/Supervisor/Remote/RemoteTaskSupervisor.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/GlobalData/Tasks/Supervisor/Remote/RemoteTaskSupervisor.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/GlobalData/Tasks/Supervisor/Remote/ServerRunSupervisor.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/GlobalData/Tasks/Supervisor/Remote/ServerRunSupervisor.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/Common/TestingPlanner.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/Common/TestingPlanner.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/DVM/TestsSupervisor_2022.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/DVM/TestsSupervisor_2022.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/DVM/UserConnection.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/DVM/UserConnection.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/All/RemoteInitialiseUser.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/All/RemoteInitialiseUser.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/files/Planner/Planner.cpp" beforeDir="false" afterPath="$PROJECT_DIR$/src/files/Planner/Planner.cpp" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/files/Planner/RunTask.h" beforeDir="false" afterPath="$PROJECT_DIR$/src/files/Planner/RunTask.h" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/SSH/ConnectionPass.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/SSH/ConnectionPass.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/files/Planner/Supervisor.h" beforeDir="false" afterPath="$PROJECT_DIR$/src/files/Planner/Supervisor.h" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/files/Planner/Task.h" beforeDir="false" afterPath="$PROJECT_DIR$/src/files/Planner/Task.h" afterDir="false" />
</list>

2
properties
View File

@@ -25,7 +25,7 @@
"InstructionPath": "",
"PerformanceAnalyzerPath": "",
"ComponentsBackUpsCount": 10,
"TestingKernels": 16,
"TestingKernels": 28,
"AutoCheckTesting": false,
"CheckTestingIntervalSeconds": 10,
"EmailOnTestingProgress": true

2
src/GlobalData/Tasks/Supervisor/Remote/RemoteCompilationSupervisor.java
View File

@@ -37,7 +37,7 @@ public class RemoteCompilationSupervisor extends RemoteTaskSupervisor<Compilatio
}
@Override
protected void ValidateTaskResults() throws Exception {
if (pass.Exists(getRemoteProject().full_name, getBinary().name)) {
if (pass.Exists(getBinary())) {
RemoteFile renamed_binary = new RemoteFile(getRemoteProject().full_name, Utils.getDateName("spf_" + getBinary().name));
pass.sftpChannel.rename(getBinary().full_name, renamed_binary.full_name);
task.binary_name = renamed_binary.name;

12
src/GlobalData/Tasks/Supervisor/Remote/RemoteTaskSupervisor.java
View File

@@ -42,15 +42,17 @@ public abstract class RemoteTaskSupervisor<T extends Task> extends TaskSuperviso
}
@Override
protected void CheckTask() throws Exception {
if (pass.Exists(getRemoteProject().full_name, Constants.DONE))
RemoteFile DONE = new RemoteFile(getRemoteProject(), Constants.DONE);
RemoteFile TIMEOUT = new RemoteFile(getRemoteProject(), Constants.TIMEOUT);
if (pass.Exists(DONE))
task.state = TaskState.Finished;
else if (pass.Exists(getRemoteProject().full_name, Constants.TIMEOUT))
else if (pass.Exists(TIMEOUT))
task.state = TaskState.AbortedByTimeout;
}
@Override
protected void AchieveResults() throws Exception {
pass.tryGetSingleFile(getRemoteOutput(), task.getOutputFile(), 10240);
pass.tryGetSingleFile(getRemoteErrors(), task.getErrorsFile(), 10240);
pass.tryGetSingleFileWithMaxSize(getRemoteOutput(), task.getOutputFile(), 10240);
pass.tryGetSingleFileWithMaxSize(getRemoteErrors(), task.getErrorsFile(), 10240);
}
@Override
protected void AbortTask() throws Exception {
@@ -58,7 +60,7 @@ public abstract class RemoteTaskSupervisor<T extends Task> extends TaskSuperviso
}
@Override
protected void CalculatePerformanceTime() throws Exception {
if (pass.tryGetSingleFile(getRemoteTime(), task.getTimeFile(), 0))
if (pass.tryGetSingleFileWithMaxSize(getRemoteTime(), task.getTimeFile(), 0))
task.RefreshTime();
}
protected String getStartCommand() {

2
src/GlobalData/Tasks/Supervisor/Remote/ServerRunSupervisor.java
View File

@@ -71,7 +71,7 @@ public class ServerRunSupervisor extends RemoteTaskSupervisor<RunTask> {
}
if (!task.last_sts_name.isEmpty()) {
RemoteFile remote_sts = new RemoteFile(getRemoteProject().full_name, task.last_sts_name);
task.hasDvmSts = pass.tryGetSingleFile(remote_sts, task.getLocalStsFile(), 10240);
task.hasDvmSts = pass.tryGetSingleFileWithMaxSize(remote_sts, task.getLocalStsFile(), 10240);
}
}
}

11
src/TestingSystem/Common/TestingPlanner.java
View File

@@ -6,14 +6,13 @@ import GlobalData.User.User;
import Repository.EmailMessage;
import Repository.Server.ServerCode;
import Repository.Server.ServerExchangeUnit_2021;
import TestingSystem.DVM.UserConnection;
import TestingSystem.SAPFOR.SapforTasksPackage.SapforTasksPackage;
import TestingSystem.SAPFOR.SapforTasksPackageSupervisor.SapforTasksPackageSupervisor;
import TestingSystem.DVM.Tasks.TestCompilationTask;
import TestingSystem.DVM.Tasks.TestTask;
import TestingSystem.DVM.TasksPackage.TasksPackage;
import TestingSystem.DVM.TasksPackage.TasksPackageState;
import TestingSystem.DVM.TestsSupervisor_2022;
import TestingSystem.DVM.UserConnection;
import TestingSystem.SAPFOR.SapforTasksPackage.SapforTasksPackage;
import TestingSystem.SAPFOR.SapforTasksPackageSupervisor.SapforTasksPackageSupervisor;
import Visual_DVM_2021.Passes.PassException;
import Visual_DVM_2021.Passes.SSH.ConnectionPass;
import Visual_DVM_2021.Passes.Server.TestingSystemPass;
@@ -39,10 +38,6 @@ public class TestingPlanner {
//----------
SapforTasksPackage sapforTasksPackage = null;
//----------
public void UpdateTask(TestTask task_in) throws Exception {
task_in.ChangeDate = new Date().getTime();
ServerCommand(ServerCode.EditAccountObject, task_in);
}
public void UpdatePackage() throws Exception {
tasksPackage.ChangeDate = new Date().getTime();
ServerCommand(ServerCode.EditAccountObject, tasksPackage);

14
src/TestingSystem/DVM/TestsSupervisor_2022.java
View File

@@ -12,7 +12,6 @@ import TestingSystem.DVM.Tasks.TestRunTask;
import TestingSystem.DVM.Tasks.TestTask;
import TestingSystem.DVM.TasksPackage.TasksPackage;
import TestingSystem.DVM.TasksPackage.TasksPackageState;
import Visual_DVM_2021.Passes.All.UnzipFolderPass;
import com.jcraft.jsch.ChannelSftp;
import javafx.util.Pair;
import org.apache.commons.io.FileUtils;
@@ -140,11 +139,12 @@ public class TestsSupervisor_2022 {
);
connection.ShellCommand(plannerStartCommand);
RemoteFile PID = new RemoteFile(packageRemoteWorkspace, "PID");
while (!connection.Exists(packageRemoteWorkspace.full_name, "STARTED")) {
RemoteFile STARTED = new RemoteFile(packageRemoteWorkspace, "STARTED");
while (!connection.Exists(STARTED)) {
System.out.println("waiting for package start...");
Utils.sleep(1000);
}
if (connection.Exists(packageRemoteWorkspace.full_name, "PID")) {
if (connection.Exists(PID)) {
tasksPackage.pid = connection.readFromFile(PID);
}
}
@@ -170,13 +170,15 @@ public class TestsSupervisor_2022 {
}
public void DownloadResults() throws Exception {
Utils.CheckDirectory(packageLocalWorkspace);
/*
RemoteFile remote_results_archive = new RemoteFile(packageRemoteWorkspace.full_name, "results.zip");
File results_archive = new File(packageLocalWorkspace, "results.zip");
if (connection.Exists(packageRemoteWorkspace.full_name, "results.zip")) {
connection.getSingleFile(remote_results_archive, results_archive, 0);
if (connection.Exists()) {
connection.getSingleFile(remote_results_archive, results_archive);
UnzipFolderPass unzipFolderPass = new UnzipFolderPass();
unzipFolderPass.Do(results_archive.getAbsolutePath(), packageLocalWorkspace.getAbsolutePath(), false);
}
*/
}
public void AnalyseResults() throws Exception {
System.out.println("analysing results");
@@ -248,7 +250,7 @@ public class TestsSupervisor_2022 {
if (stateFile.exists()) {
String stateText = FileUtils.readFileToString(stateFile, Charset.defaultCharset()).replace("\n", "");
testTask.state = TaskState.valueOf(stateText);
} else testTask.state = TaskState.InternalError; //поменять на то что состояние не найдено. ?
} else testTask.state = TaskState.InternalError;
return true;
}
return false;

58
src/TestingSystem/DVM/UserConnection.java
View File

@@ -7,10 +7,7 @@ import GlobalData.Machine.Machine;
import GlobalData.RemoteFile.RemoteFile;
import GlobalData.User.User;
import Visual_DVM_2021.Passes.PassException;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.ChannelShell;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.Session;
import com.jcraft.jsch.*;
import java.io.*;
import java.nio.charset.StandardCharsets;
@@ -108,8 +105,8 @@ public class UserConnection {
System.gc();
}
//--
//из за мусора результатом пользоваться в общем случае невозможно.
//todo из за мусора результатом пользоваться в общем случае невозможно.
//следует перенаправлять вывод в какой нибудь временный файл на сервере.
public String ShellCommand(String command) throws Exception {
StringBuilder result = new StringBuilder();
// System.out.println("command=" + Utils.Brackets(command));
@@ -122,43 +119,26 @@ public class UserConnection {
// System.out.println("res="+Utils.Brackets(res));
return (data.length > 0) ? data[data.length - 1] : result.toString();
}
//--
//тут имя файла короткое.
public boolean Exists(String folder, String name) throws Exception {
Vector<ChannelSftp.LsEntry> files = sftpChannel.ls(folder);
for (ChannelSftp.LsEntry file : files) {
file.getAttrs().getSize();
if (file.getFilename().equals(name)) {
return true;
}
}
return false;
}
//--
public void getSingleFile(String src, String dst) throws Exception {
sftpChannel.get(src, dst);
}
public long getFileKBSize(String path) throws Exception{
public long getFileKBSize(String path) throws Exception {
long size = sftpChannel.lstat(path).getSize();
return size/1024;
return size / 1024;
}
public boolean getSingleFile(RemoteFile src, File dst, int maxSize) throws Exception {
if (Exists(src.parent, src.name)) {
public void getSingleFileWithMaxSize(RemoteFile src, File dst, int maxSize) throws Exception {
if ((maxSize == 0) || getFileKBSize(src.full_name) <= maxSize) {
getSingleFile(src.full_name, dst.getAbsolutePath());
return true;
} else {
Utils.WriteToFile(dst, "Размер файла превышает " + maxSize + " KB.\n" + "Файл не загружен. Его можно просмотреть на машине по адресу\n" + Utils.Brackets(src.full_name));
}
}
return false;
}
public void putSingleFile(File src, RemoteFile dst) throws Exception {
sftpChannel.put(src.getAbsolutePath(), dst.full_name);
}
//-
public void MKDIR(RemoteFile dir) throws Exception {
if (!Exists(dir.parent, dir.name)) sftpChannel.mkdir(dir.full_name);
if (!Exists(dir)) sftpChannel.mkdir(dir.full_name);
}
//-
public void RMDIR(String dir) throws Exception {
@@ -210,12 +190,6 @@ public class UserConnection {
}
}
}
/*
public void copy(RemoteFile src, RemoteFile dst) throws Exception {
ShellCommand("cp " + Utils.DQuotes(src.full_name) + " " + Utils.DQuotes(dst.full_name));
}
*/
//-------
public void writeToFile(String text, RemoteFile dst) throws Exception {
sftpChannel.put(new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8)), dst.full_name);
sftpChannel.chmod(0777, dst.full_name);
@@ -225,4 +199,22 @@ public class UserConnection {
sftpChannel.get(src.full_name, outputStream);
return outputStream.toString(StandardCharsets.UTF_8.name());
}
//--
public boolean Exists(String file_full_name) throws Exception {
try {
sftpChannel.lstat(file_full_name);
return true;
} catch (SftpException e) {
if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) {
// file doesn't exist
return false;
} else {
// something else went wrong
throw e;
}
}
}
public boolean Exists(RemoteFile file) throws Exception {
return Exists(file.full_name);
}
}

6
src/Visual_DVM_2021/Passes/All/InstallServerSapfor.java
View File

@@ -59,11 +59,12 @@ public class InstallServerSapfor extends ConnectionPass<Object> {
ShowMessage1("Сборка SAPFOR...");
//-
RemoteFile repo_bin = new RemoteFile(repoSapforHome.full_name, "Sapfor_F");
if (Exists(repoSapforHome.full_name, repo_bin.name))
if (Exists(repo_bin))
sftpChannel.rm(repo_bin.full_name);
//--
performScript(repoSapforHome, "cmake ../", "make -j 4");
result = Exists(repoSapforHome.full_name, "Sapfor_F");
RemoteFile repoSapfor = new RemoteFile(repoSapforHome, "Sapfor_F");
result = Exists(repoSapfor);
if (result) {
RemoteFile sapforsDirectory = new RemoteFile(testingSystemHome.full_name, "Sapfors", true);
//создать папку. Для того чтобы скопировать из репозитория.
@@ -116,6 +117,5 @@ public class InstallServerSapfor extends ConnectionPass<Object> {
protected void performDone() throws Exception {
passes.get(PassCode_2021.PublishServerSapfor).Do(serverSapfor);
}
}

2
src/Visual_DVM_2021/Passes/All/RemoteInitialiseUser.java
View File

@@ -69,7 +69,7 @@ public class RemoteInitialiseUser extends CurrentConnectionPass<String> {
"cd " + Utils.DQuotes(sftpChannel.pwd()), //нужны ли тут кавычки?
"g++ " + starter_code + " -o " + starter,
"g++ " + launcher_code + " -o " + launcher,
"g++ -O3 -std=c++17 Planner.cpp -o " + planner,
"g++ -O3 -std=C++17 Planner.cpp -o " + planner,
"chmod 0777 " + starter,
"chmod 0777 " + launcher,
"chmod 0777 " + planner

52
src/Visual_DVM_2021/Passes/SSH/ConnectionPass.java
View File

@@ -85,7 +85,7 @@ public abstract class ConnectionPass<T> extends Pass_2021<T> {
isConnected = true; // теперь можно прерывать метод.
if (needsInitialize()) {
RemoteFile userWorkspace = new RemoteFile(user.workspace, true);
if (!Exists(sftpChannel.getHome(), userWorkspace.name))
if (!Exists(userWorkspace))
throw new WorkspaceNotFoundException(
"Рабочее пространство пользователя " + Utils.Brackets(user.login)
+ " на машине " + Utils.Brackets(machine.getURL())
@@ -171,8 +171,8 @@ public abstract class ConnectionPass<T> extends Pass_2021<T> {
sftpChannel.get(src, dst);
}
//с проверкой.
public boolean tryGetSingleFile(RemoteFile src, File dst, int maxSize) throws Exception {
if (Exists(src.parent, src.name)) {
public boolean tryGetSingleFileWithMaxSize(RemoteFile src, File dst, int maxSize) throws Exception {
if (Exists(src)) {
if ((maxSize == 0) || (getFileKBSize(src.full_name) <= maxSize)) {
getSingleFile(src.full_name, dst.getAbsolutePath());
return true;
@@ -186,19 +186,11 @@ public abstract class ConnectionPass<T> extends Pass_2021<T> {
sftpChannel.put(src.getAbsolutePath(), dst.full_name);
}
public void tryMKDir(RemoteFile dir) throws Exception {
// System.out.print("try mkdir: '" + dir.full_name);
if (!Exists(dir.parent, dir.name)) sftpChannel.mkdir(dir.full_name);
// System.out.println("..done");
if (!Exists(dir)) sftpChannel.mkdir(dir.full_name);
}
public void tryRM(RemoteFile file) throws Exception {
// System.out.print("try remove: '" + file.full_name);
if (Exists(file.parent, file.name)) {
// System.out.print("' :exists.needs remove..");
if (Exists(file))
sftpChannel.rm(file.full_name);
// System.out.println(" +");
} else {
// System.out.println("no such file");
}
}
public void putSingleFile(String src, String dst) throws Exception {
sftpChannel.put(src, dst);
@@ -213,16 +205,6 @@ public abstract class ConnectionPass<T> extends Pass_2021<T> {
}
protected void ServerAction() throws Exception {
}
//тут имя файла короткое.
public boolean Exists(String folder, String name) throws Exception {
Vector<ChannelSftp.LsEntry> files = sftpChannel.ls(folder);
for (ChannelSftp.LsEntry file : files) {
if (file.getFilename().equals(name)) {
return true;
}
}
return false;
}
//https://losst.ru/komanda-find-v-linux#%D0%9E%D1%81%D0%BD%D0%BE%D0%B2%D0%BD%D1%8B%D0%B5_%D0%BF%D0%B0%D1%80%D0%B0%D0%BC%D0%B5%D1%82%D1%80%D1%8B_%D0%BA%D0%BE%D0%BC%D0%B0%D0%BD%D0%B4%D1%8B_find
public String getStarter() {
return String.join("/", user.workspace, modules, starter);
@@ -385,7 +367,7 @@ public abstract class ConnectionPass<T> extends Pass_2021<T> {
//скорее всего,временные методы. они есть в UserConnection, при удаленном запуске тестирования
//--------------------------------------------------------------------------------
public void MKDIR(RemoteFile dir) throws Exception {
if (!Exists(dir.parent, dir.name)) sftpChannel.mkdir(dir.full_name);
if (!Exists(dir)) sftpChannel.mkdir(dir.full_name);
}
//--
public Pair<RemoteFile, RemoteFile> performScript(RemoteFile directory, String... commands) throws Exception {
@@ -398,7 +380,7 @@ public abstract class ConnectionPass<T> extends Pass_2021<T> {
files.add(out);
files.add(err);
for (RemoteFile file : files) {
if (Exists(directory.full_name, file.name))
if (Exists(directory))
sftpChannel.rm(file.full_name);
}
//--
@@ -406,9 +388,27 @@ public abstract class ConnectionPass<T> extends Pass_2021<T> {
//--
ShellCommand("cd " + Utils.DQuotes(directory.full_name),
script_file.full_name + " 1>" + Constants.out_file + " 2>" + Constants.err_file);
return new Pair<>(out, err);
}
//--
//--
public boolean Exists(String file_full_name) throws Exception {
try {
sftpChannel.stat(file_full_name);
return true;
} catch (SftpException e) {
if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) {
// file doesn't exist
return false;
} else {
// something else went wrong
throw e;
}
}
}
public boolean Exists(RemoteFile file) throws Exception {
return Exists(file.full_name);
}
}

10
src/files/Planner/Supervisor.h
View File

@@ -126,7 +126,7 @@ public:
String pathRes("results");
Utils::Mkdir(pathRes);
//string buf;
string buf;
while (activeTasks) {
long oldActiveTasks = activeTasks;
@@ -174,7 +174,7 @@ public:
busyKernels -= task->getKernels();
printf(" done task with %d kernels and id %ld\n", task->getKernels(), task->getId());
//buf += to_string(task->getId()) + " " + string(task->printState().getCharArray()) + " " + to_string(task->getTotalTime()) + "\n";
buf += to_string(task->getId()) + " " + string(task->printState().getCharArray()) + " " + to_string(task->getTotalTime()) + "\n";
task->copyResults(pathRes);
continue;
}
@@ -187,10 +187,10 @@ public:
changeState();
//String outFile(pathRes + "/info.txt");
//File tmp(outFile, String(buf.c_str()));
String outFile(pathRes + "/info.txt");
File tmp(outFile, String(buf.c_str()));
Utils::ZipFolder(pathRes, pathRes + ".zip");
//Utils::ZipFolder(pathRes, pathRes + ".zip");
}
virtual void Finalize() { }

4
src/files/Planner/Task.h
View File

@@ -201,10 +201,10 @@ public:
virtual String copyResults(const String& pathRes) {
String resultPath(packageWorkspace + "/" + pathRes + "/" + getId());
Utils::Mkdir(resultPath);
Utils::Copy(workspace + "/TaskState", resultPath + "/TaskState");
//Utils::Copy(workspace + "/TaskState", resultPath + "/TaskState");
Utils::Copy(workspace + "/out.txt", resultPath + "/out.txt");
Utils::Copy(workspace + "/err.txt", resultPath + "/err.txt");
Utils::Copy(workspace + "/total_time", resultPath + "/total_time");
//Utils::Copy(workspace + "/total_time", resultPath + "/total_time");
return resultPath;
}
};