From 163552d74fab9727bca0ca64ddadd0783e3200ee Mon Sep 17 00:00:00 2001 From: 02090095 Date: Mon, 4 Dec 2023 14:42:36 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9E=D0=BF=D1=82=D0=B8=D0=BC=D0=B8=D0=B7?= =?UTF-8?q?=D0=B0=D1=86=D0=B8=D1=8F=20=D0=BA=D0=BE=D0=BC=D0=B0=D0=BD=D0=B4?= =?UTF-8?q?=D1=8B=20Exists=20=D0=B4=D0=BB=D1=8F=20SSH?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .idea/workspace.xml | 10 +-- properties | 2 +- .../Remote/RemoteCompilationSupervisor.java | 2 +- .../Remote/RemoteTaskSupervisor.java | 12 ++-- .../Remote/ServerRunSupervisor.java | 2 +- src/TestingSystem/Common/TestingPlanner.java | 11 +--- .../DVM/TestsSupervisor_2022.java | 14 ++-- src/TestingSystem/DVM/UserConnection.java | 66 ++++++++----------- .../Passes/All/InstallServerSapfor.java | 6 +- .../Passes/All/RemoteInitialiseUser.java | 2 +- .../Passes/SSH/ConnectionPass.java | 52 +++++++-------- src/files/Planner/Supervisor.h | 10 +-- src/files/Planner/Task.h | 4 +- 13 files changed, 93 insertions(+), 100 deletions(-) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 0bf193c2..77881351 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -8,12 +8,14 @@ - - + + + + + - - + diff --git a/properties b/properties index 05cc0d0d..1570469b 100644 --- a/properties +++ b/properties @@ -25,7 +25,7 @@ "InstructionPath": "", "PerformanceAnalyzerPath": "", "ComponentsBackUpsCount": 10, - "TestingKernels": 16, + "TestingKernels": 28, "AutoCheckTesting": false, "CheckTestingIntervalSeconds": 10, "EmailOnTestingProgress": true diff --git a/src/GlobalData/Tasks/Supervisor/Remote/RemoteCompilationSupervisor.java b/src/GlobalData/Tasks/Supervisor/Remote/RemoteCompilationSupervisor.java index 38f110a0..450dbbbb 100644 --- a/src/GlobalData/Tasks/Supervisor/Remote/RemoteCompilationSupervisor.java +++ b/src/GlobalData/Tasks/Supervisor/Remote/RemoteCompilationSupervisor.java @@ -37,7 +37,7 @@ public class RemoteCompilationSupervisor extends RemoteTaskSupervisor extends TaskSuperviso } @Override protected void CheckTask() throws Exception { - if (pass.Exists(getRemoteProject().full_name, Constants.DONE)) + RemoteFile DONE = new RemoteFile(getRemoteProject(), Constants.DONE); + RemoteFile TIMEOUT = new RemoteFile(getRemoteProject(), Constants.TIMEOUT); + if (pass.Exists(DONE)) task.state = TaskState.Finished; - else if (pass.Exists(getRemoteProject().full_name, Constants.TIMEOUT)) + else if (pass.Exists(TIMEOUT)) task.state = TaskState.AbortedByTimeout; } @Override protected void AchieveResults() throws Exception { - pass.tryGetSingleFile(getRemoteOutput(), task.getOutputFile(), 10240); - pass.tryGetSingleFile(getRemoteErrors(), task.getErrorsFile(), 10240); + pass.tryGetSingleFileWithMaxSize(getRemoteOutput(), task.getOutputFile(), 10240); + pass.tryGetSingleFileWithMaxSize(getRemoteErrors(), task.getErrorsFile(), 10240); } @Override protected void AbortTask() throws Exception { @@ -58,7 +60,7 @@ public abstract class RemoteTaskSupervisor extends TaskSuperviso } @Override protected void CalculatePerformanceTime() throws Exception { - if (pass.tryGetSingleFile(getRemoteTime(), task.getTimeFile(), 0)) + if (pass.tryGetSingleFileWithMaxSize(getRemoteTime(), task.getTimeFile(), 0)) task.RefreshTime(); } protected String getStartCommand() { diff --git a/src/GlobalData/Tasks/Supervisor/Remote/ServerRunSupervisor.java b/src/GlobalData/Tasks/Supervisor/Remote/ServerRunSupervisor.java index 4fe8cefe..cea94a50 100644 --- a/src/GlobalData/Tasks/Supervisor/Remote/ServerRunSupervisor.java +++ b/src/GlobalData/Tasks/Supervisor/Remote/ServerRunSupervisor.java @@ -71,7 +71,7 @@ public class ServerRunSupervisor extends RemoteTaskSupervisor { } if (!task.last_sts_name.isEmpty()) { RemoteFile remote_sts = new RemoteFile(getRemoteProject().full_name, task.last_sts_name); - task.hasDvmSts = pass.tryGetSingleFile(remote_sts, task.getLocalStsFile(), 10240); + task.hasDvmSts = pass.tryGetSingleFileWithMaxSize(remote_sts, task.getLocalStsFile(), 10240); } } } diff --git a/src/TestingSystem/Common/TestingPlanner.java b/src/TestingSystem/Common/TestingPlanner.java index c49ca0ed..d77c2f2c 100644 --- a/src/TestingSystem/Common/TestingPlanner.java +++ b/src/TestingSystem/Common/TestingPlanner.java @@ -6,14 +6,13 @@ import GlobalData.User.User; import Repository.EmailMessage; import Repository.Server.ServerCode; import Repository.Server.ServerExchangeUnit_2021; -import TestingSystem.DVM.UserConnection; -import TestingSystem.SAPFOR.SapforTasksPackage.SapforTasksPackage; -import TestingSystem.SAPFOR.SapforTasksPackageSupervisor.SapforTasksPackageSupervisor; import TestingSystem.DVM.Tasks.TestCompilationTask; -import TestingSystem.DVM.Tasks.TestTask; import TestingSystem.DVM.TasksPackage.TasksPackage; import TestingSystem.DVM.TasksPackage.TasksPackageState; import TestingSystem.DVM.TestsSupervisor_2022; +import TestingSystem.DVM.UserConnection; +import TestingSystem.SAPFOR.SapforTasksPackage.SapforTasksPackage; +import TestingSystem.SAPFOR.SapforTasksPackageSupervisor.SapforTasksPackageSupervisor; import Visual_DVM_2021.Passes.PassException; import Visual_DVM_2021.Passes.SSH.ConnectionPass; import Visual_DVM_2021.Passes.Server.TestingSystemPass; @@ -39,10 +38,6 @@ public class TestingPlanner { //---------- SapforTasksPackage sapforTasksPackage = null; //---------- - public void UpdateTask(TestTask task_in) throws Exception { - task_in.ChangeDate = new Date().getTime(); - ServerCommand(ServerCode.EditAccountObject, task_in); - } public void UpdatePackage() throws Exception { tasksPackage.ChangeDate = new Date().getTime(); ServerCommand(ServerCode.EditAccountObject, tasksPackage); diff --git a/src/TestingSystem/DVM/TestsSupervisor_2022.java b/src/TestingSystem/DVM/TestsSupervisor_2022.java index 8cfa62b3..49a3d619 100644 --- a/src/TestingSystem/DVM/TestsSupervisor_2022.java +++ b/src/TestingSystem/DVM/TestsSupervisor_2022.java @@ -12,7 +12,6 @@ import TestingSystem.DVM.Tasks.TestRunTask; import TestingSystem.DVM.Tasks.TestTask; import TestingSystem.DVM.TasksPackage.TasksPackage; import TestingSystem.DVM.TasksPackage.TasksPackageState; -import Visual_DVM_2021.Passes.All.UnzipFolderPass; import com.jcraft.jsch.ChannelSftp; import javafx.util.Pair; import org.apache.commons.io.FileUtils; @@ -140,11 +139,12 @@ public class TestsSupervisor_2022 { ); connection.ShellCommand(plannerStartCommand); RemoteFile PID = new RemoteFile(packageRemoteWorkspace, "PID"); - while (!connection.Exists(packageRemoteWorkspace.full_name, "STARTED")) { + RemoteFile STARTED = new RemoteFile(packageRemoteWorkspace, "STARTED"); + while (!connection.Exists(STARTED)) { System.out.println("waiting for package start..."); Utils.sleep(1000); } - if (connection.Exists(packageRemoteWorkspace.full_name, "PID")) { + if (connection.Exists(PID)) { tasksPackage.pid = connection.readFromFile(PID); } } @@ -170,13 +170,15 @@ public class TestsSupervisor_2022 { } public void DownloadResults() throws Exception { Utils.CheckDirectory(packageLocalWorkspace); + /* RemoteFile remote_results_archive = new RemoteFile(packageRemoteWorkspace.full_name, "results.zip"); File results_archive = new File(packageLocalWorkspace, "results.zip"); - if (connection.Exists(packageRemoteWorkspace.full_name, "results.zip")) { - connection.getSingleFile(remote_results_archive, results_archive, 0); + if (connection.Exists()) { + connection.getSingleFile(remote_results_archive, results_archive); UnzipFolderPass unzipFolderPass = new UnzipFolderPass(); unzipFolderPass.Do(results_archive.getAbsolutePath(), packageLocalWorkspace.getAbsolutePath(), false); } + */ } public void AnalyseResults() throws Exception { System.out.println("analysing results"); @@ -248,7 +250,7 @@ public class TestsSupervisor_2022 { if (stateFile.exists()) { String stateText = FileUtils.readFileToString(stateFile, Charset.defaultCharset()).replace("\n", ""); testTask.state = TaskState.valueOf(stateText); - } else testTask.state = TaskState.InternalError; //поменять на то что состояние не найдено. ? + } else testTask.state = TaskState.InternalError; return true; } return false; diff --git a/src/TestingSystem/DVM/UserConnection.java b/src/TestingSystem/DVM/UserConnection.java index 11feec01..af99fa0d 100644 --- a/src/TestingSystem/DVM/UserConnection.java +++ b/src/TestingSystem/DVM/UserConnection.java @@ -7,10 +7,7 @@ import GlobalData.Machine.Machine; import GlobalData.RemoteFile.RemoteFile; import GlobalData.User.User; import Visual_DVM_2021.Passes.PassException; -import com.jcraft.jsch.ChannelSftp; -import com.jcraft.jsch.ChannelShell; -import com.jcraft.jsch.JSch; -import com.jcraft.jsch.Session; +import com.jcraft.jsch.*; import java.io.*; import java.nio.charset.StandardCharsets; @@ -108,8 +105,8 @@ public class UserConnection { System.gc(); } //-- - - //из за мусора результатом пользоваться в общем случае невозможно. + //todo из за мусора результатом пользоваться в общем случае невозможно. + //следует перенаправлять вывод в какой нибудь временный файл на сервере. public String ShellCommand(String command) throws Exception { StringBuilder result = new StringBuilder(); // System.out.println("command=" + Utils.Brackets(command)); @@ -122,43 +119,26 @@ public class UserConnection { // System.out.println("res="+Utils.Brackets(res)); return (data.length > 0) ? data[data.length - 1] : result.toString(); } - //-- - //тут имя файла короткое. - public boolean Exists(String folder, String name) throws Exception { - Vector files = sftpChannel.ls(folder); - for (ChannelSftp.LsEntry file : files) { - file.getAttrs().getSize(); - if (file.getFilename().equals(name)) { - return true; - } - } - return false; - } - //-- public void getSingleFile(String src, String dst) throws Exception { sftpChannel.get(src, dst); } - public long getFileKBSize(String path) throws Exception{ + public long getFileKBSize(String path) throws Exception { long size = sftpChannel.lstat(path).getSize(); - return size/1024; + return size / 1024; } - public boolean getSingleFile(RemoteFile src, File dst, int maxSize) throws Exception { - if (Exists(src.parent, src.name)) { - if ((maxSize == 0) || getFileKBSize(src.full_name) <= maxSize) { - getSingleFile(src.full_name, dst.getAbsolutePath()); - return true; - } else { - Utils.WriteToFile(dst, "Размер файла превышает " + maxSize + " KB.\n" + "Файл не загружен. Его можно просмотреть на машине по адресу\n" + Utils.Brackets(src.full_name)); - } + public void getSingleFileWithMaxSize(RemoteFile src, File dst, int maxSize) throws Exception { + if ((maxSize == 0) || getFileKBSize(src.full_name) <= maxSize) { + getSingleFile(src.full_name, dst.getAbsolutePath()); + } else { + Utils.WriteToFile(dst, "Размер файла превышает " + maxSize + " KB.\n" + "Файл не загружен. Его можно просмотреть на машине по адресу\n" + Utils.Brackets(src.full_name)); } - return false; } public void putSingleFile(File src, RemoteFile dst) throws Exception { sftpChannel.put(src.getAbsolutePath(), dst.full_name); } //- public void MKDIR(RemoteFile dir) throws Exception { - if (!Exists(dir.parent, dir.name)) sftpChannel.mkdir(dir.full_name); + if (!Exists(dir)) sftpChannel.mkdir(dir.full_name); } //- public void RMDIR(String dir) throws Exception { @@ -210,12 +190,6 @@ public class UserConnection { } } } - /* - public void copy(RemoteFile src, RemoteFile dst) throws Exception { - ShellCommand("cp " + Utils.DQuotes(src.full_name) + " " + Utils.DQuotes(dst.full_name)); - } - */ - //------- public void writeToFile(String text, RemoteFile dst) throws Exception { sftpChannel.put(new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8)), dst.full_name); sftpChannel.chmod(0777, dst.full_name); @@ -225,4 +199,22 @@ public class UserConnection { sftpChannel.get(src.full_name, outputStream); return outputStream.toString(StandardCharsets.UTF_8.name()); } + //-- + public boolean Exists(String file_full_name) throws Exception { + try { + sftpChannel.lstat(file_full_name); + return true; + } catch (SftpException e) { + if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) { + // file doesn't exist + return false; + } else { + // something else went wrong + throw e; + } + } + } + public boolean Exists(RemoteFile file) throws Exception { + return Exists(file.full_name); + } } diff --git a/src/Visual_DVM_2021/Passes/All/InstallServerSapfor.java b/src/Visual_DVM_2021/Passes/All/InstallServerSapfor.java index 4ae251cb..5eb431fb 100644 --- a/src/Visual_DVM_2021/Passes/All/InstallServerSapfor.java +++ b/src/Visual_DVM_2021/Passes/All/InstallServerSapfor.java @@ -59,11 +59,12 @@ public class InstallServerSapfor extends ConnectionPass { ShowMessage1("Сборка SAPFOR..."); //- RemoteFile repo_bin = new RemoteFile(repoSapforHome.full_name, "Sapfor_F"); - if (Exists(repoSapforHome.full_name, repo_bin.name)) + if (Exists(repo_bin)) sftpChannel.rm(repo_bin.full_name); //-- performScript(repoSapforHome, "cmake ../", "make -j 4"); - result = Exists(repoSapforHome.full_name, "Sapfor_F"); + RemoteFile repoSapfor = new RemoteFile(repoSapforHome, "Sapfor_F"); + result = Exists(repoSapfor); if (result) { RemoteFile sapforsDirectory = new RemoteFile(testingSystemHome.full_name, "Sapfors", true); //создать папку. Для того чтобы скопировать из репозитория. @@ -116,6 +117,5 @@ public class InstallServerSapfor extends ConnectionPass { protected void performDone() throws Exception { passes.get(PassCode_2021.PublishServerSapfor).Do(serverSapfor); } - } diff --git a/src/Visual_DVM_2021/Passes/All/RemoteInitialiseUser.java b/src/Visual_DVM_2021/Passes/All/RemoteInitialiseUser.java index e4ff5fa4..15391b6d 100644 --- a/src/Visual_DVM_2021/Passes/All/RemoteInitialiseUser.java +++ b/src/Visual_DVM_2021/Passes/All/RemoteInitialiseUser.java @@ -69,7 +69,7 @@ public class RemoteInitialiseUser extends CurrentConnectionPass { "cd " + Utils.DQuotes(sftpChannel.pwd()), //нужны ли тут кавычки? "g++ " + starter_code + " -o " + starter, "g++ " + launcher_code + " -o " + launcher, - "g++ -O3 -std=c++17 Planner.cpp -o " + planner, + "g++ -O3 -std=C++17 Planner.cpp -o " + planner, "chmod 0777 " + starter, "chmod 0777 " + launcher, "chmod 0777 " + planner diff --git a/src/Visual_DVM_2021/Passes/SSH/ConnectionPass.java b/src/Visual_DVM_2021/Passes/SSH/ConnectionPass.java index 8b9c104a..1c5bfc99 100644 --- a/src/Visual_DVM_2021/Passes/SSH/ConnectionPass.java +++ b/src/Visual_DVM_2021/Passes/SSH/ConnectionPass.java @@ -85,7 +85,7 @@ public abstract class ConnectionPass extends Pass_2021 { isConnected = true; // теперь можно прерывать метод. if (needsInitialize()) { RemoteFile userWorkspace = new RemoteFile(user.workspace, true); - if (!Exists(sftpChannel.getHome(), userWorkspace.name)) + if (!Exists(userWorkspace)) throw new WorkspaceNotFoundException( "Рабочее пространство пользователя " + Utils.Brackets(user.login) + " на машине " + Utils.Brackets(machine.getURL()) @@ -171,8 +171,8 @@ public abstract class ConnectionPass extends Pass_2021 { sftpChannel.get(src, dst); } //с проверкой. - public boolean tryGetSingleFile(RemoteFile src, File dst, int maxSize) throws Exception { - if (Exists(src.parent, src.name)) { + public boolean tryGetSingleFileWithMaxSize(RemoteFile src, File dst, int maxSize) throws Exception { + if (Exists(src)) { if ((maxSize == 0) || (getFileKBSize(src.full_name) <= maxSize)) { getSingleFile(src.full_name, dst.getAbsolutePath()); return true; @@ -186,19 +186,11 @@ public abstract class ConnectionPass extends Pass_2021 { sftpChannel.put(src.getAbsolutePath(), dst.full_name); } public void tryMKDir(RemoteFile dir) throws Exception { - // System.out.print("try mkdir: '" + dir.full_name); - if (!Exists(dir.parent, dir.name)) sftpChannel.mkdir(dir.full_name); - // System.out.println("..done"); + if (!Exists(dir)) sftpChannel.mkdir(dir.full_name); } public void tryRM(RemoteFile file) throws Exception { - // System.out.print("try remove: '" + file.full_name); - if (Exists(file.parent, file.name)) { - // System.out.print("' :exists.needs remove.."); + if (Exists(file)) sftpChannel.rm(file.full_name); - // System.out.println(" +"); - } else { - // System.out.println("no such file"); - } } public void putSingleFile(String src, String dst) throws Exception { sftpChannel.put(src, dst); @@ -213,16 +205,6 @@ public abstract class ConnectionPass extends Pass_2021 { } protected void ServerAction() throws Exception { } - //тут имя файла короткое. - public boolean Exists(String folder, String name) throws Exception { - Vector files = sftpChannel.ls(folder); - for (ChannelSftp.LsEntry file : files) { - if (file.getFilename().equals(name)) { - return true; - } - } - return false; - } //https://losst.ru/komanda-find-v-linux#%D0%9E%D1%81%D0%BD%D0%BE%D0%B2%D0%BD%D1%8B%D0%B5_%D0%BF%D0%B0%D1%80%D0%B0%D0%BC%D0%B5%D1%82%D1%80%D1%8B_%D0%BA%D0%BE%D0%BC%D0%B0%D0%BD%D0%B4%D1%8B_find public String getStarter() { return String.join("/", user.workspace, modules, starter); @@ -385,7 +367,7 @@ public abstract class ConnectionPass extends Pass_2021 { //скорее всего,временные методы. они есть в UserConnection, при удаленном запуске тестирования //-------------------------------------------------------------------------------- public void MKDIR(RemoteFile dir) throws Exception { - if (!Exists(dir.parent, dir.name)) sftpChannel.mkdir(dir.full_name); + if (!Exists(dir)) sftpChannel.mkdir(dir.full_name); } //-- public Pair performScript(RemoteFile directory, String... commands) throws Exception { @@ -398,7 +380,7 @@ public abstract class ConnectionPass extends Pass_2021 { files.add(out); files.add(err); for (RemoteFile file : files) { - if (Exists(directory.full_name, file.name)) + if (Exists(directory)) sftpChannel.rm(file.full_name); } //-- @@ -406,9 +388,27 @@ public abstract class ConnectionPass extends Pass_2021 { //-- ShellCommand("cd " + Utils.DQuotes(directory.full_name), script_file.full_name + " 1>" + Constants.out_file + " 2>" + Constants.err_file); - return new Pair<>(out, err); } + //-- + //-- + public boolean Exists(String file_full_name) throws Exception { + try { + sftpChannel.stat(file_full_name); + return true; + } catch (SftpException e) { + if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) { + // file doesn't exist + return false; + } else { + // something else went wrong + throw e; + } + } + } + public boolean Exists(RemoteFile file) throws Exception { + return Exists(file.full_name); + } } diff --git a/src/files/Planner/Supervisor.h b/src/files/Planner/Supervisor.h index 767bdbce..f60c0092 100644 --- a/src/files/Planner/Supervisor.h +++ b/src/files/Planner/Supervisor.h @@ -126,7 +126,7 @@ public: String pathRes("results"); Utils::Mkdir(pathRes); - //string buf; + string buf; while (activeTasks) { long oldActiveTasks = activeTasks; @@ -174,7 +174,7 @@ public: busyKernels -= task->getKernels(); printf(" done task with %d kernels and id %ld\n", task->getKernels(), task->getId()); - //buf += to_string(task->getId()) + " " + string(task->printState().getCharArray()) + " " + to_string(task->getTotalTime()) + "\n"; + buf += to_string(task->getId()) + " " + string(task->printState().getCharArray()) + " " + to_string(task->getTotalTime()) + "\n"; task->copyResults(pathRes); continue; } @@ -187,10 +187,10 @@ public: changeState(); - //String outFile(pathRes + "/info.txt"); - //File tmp(outFile, String(buf.c_str())); + String outFile(pathRes + "/info.txt"); + File tmp(outFile, String(buf.c_str())); - Utils::ZipFolder(pathRes, pathRes + ".zip"); + //Utils::ZipFolder(pathRes, pathRes + ".zip"); } virtual void Finalize() { } diff --git a/src/files/Planner/Task.h b/src/files/Planner/Task.h index e929a055..dc463fe4 100644 --- a/src/files/Planner/Task.h +++ b/src/files/Planner/Task.h @@ -201,10 +201,10 @@ public: virtual String copyResults(const String& pathRes) { String resultPath(packageWorkspace + "/" + pathRes + "/" + getId()); Utils::Mkdir(resultPath); - Utils::Copy(workspace + "/TaskState", resultPath + "/TaskState"); + //Utils::Copy(workspace + "/TaskState", resultPath + "/TaskState"); Utils::Copy(workspace + "/out.txt", resultPath + "/out.txt"); Utils::Copy(workspace + "/err.txt", resultPath + "/err.txt"); - Utils::Copy(workspace + "/total_time", resultPath + "/total_time"); + //Utils::Copy(workspace + "/total_time", resultPath + "/total_time"); return resultPath; } };