VisualSapfor/src/TestingSystem/DVM/DVMTestingPlanner.java

package TestingSystem.DVM;
import Common.Constants;
import Common.Global;
import Common.Utils.Utils;
import GlobalData.Machine.Machine;
import GlobalData.RemoteFile.RemoteFile;
import GlobalData.Tasks.TaskState;
import GlobalData.User.User;
import ProjectData.Files.ProjectFile;
import ProjectData.LanguageName;
import Repository.Server.ServerCode;
import TestingSystem.Common.TestingPlanner;
import TestingSystem.DVM.DVMPackage.DVMPackage;
import TestingSystem.DVM.DVMTasks.DVMCompilationTask;
import TestingSystem.DVM.DVMTasks.DVMRunTask;
import TestingSystem.DVM.DVMTasks.DVMTask;
import TestingSystem.DVM.TasksPackage.TasksPackageState;
import Visual_DVM_2021.Passes.All.UnzipFolderPass;
import Visual_DVM_2021.Passes.SSH.ConnectionPass;
import javafx.util.Pair;
import org.apache.commons.io.FileUtils;

import java.io.File;
import java.io.FileFilter;
import java.nio.charset.Charset;
import java.nio.file.Paths;
import java.util.*;
public class DVMTestingPlanner extends TestingPlanner<DVMPackage> {
    //----
    LinkedHashMap<String, Machine> machines = new LinkedHashMap<>();
    //todo приделать к ним очередь? или сделать в бд тестирования список машин.
    LinkedHashMap<String, User> users = new LinkedHashMap<>();
    //-- текущие машина и пользователь.
    Machine machine = null;
    User user = null;
    //----
    protected RemoteFile packageRemoteWorkspace;
    protected File packageLocalWorkspace;
    //--
    public String getPlanner() {
        return String.join("/", user.workspace, ConnectionPass.modules, ConnectionPass.planner);
    }
    //--
    //Получить ид тестов и их папки на сервере.
    LinkedHashMap<Integer, File> getTestsFromJson() {
        LinkedHashMap<Integer, File> res = new LinkedHashMap<>();
        for (DVMCompilationTask task : testingPackage.package_json.compilationTasks) {
            if (!res.containsKey(task.test_id)) {
                res.put(task.test_id, Paths.get(Global.TestsDirectory.getAbsolutePath(), String.valueOf(task.test_id)).toFile());
            }
        }
        return res;
    }
    static LinkedHashMap<LanguageName, Vector<ProjectFile>> getTestPrograms(File test) {
        LinkedHashMap<LanguageName, Vector<ProjectFile>> res = new LinkedHashMap<>();
        //--
        res.put(LanguageName.fortran, new Vector<>());
        res.put(LanguageName.c, new Vector<>());
        res.put(LanguageName.cpp, new Vector<>());
        //--
        File[] files = test.listFiles(new FileFilter() {
            @Override
            public boolean accept(File pathname) {
                return pathname.isFile();
            }
        });
        if (files != null) {
            for (File file : files) {
                ProjectFile projectFile = new ProjectFile(new File(file.getName()));
                if (projectFile.isNotExcludedProgram())
                    res.get(projectFile.languageName).add(projectFile);
            }
        }
        return res;
    }
    static void generateForLanguage(
            String dvm_drv,
            LanguageName language,
            Vector<ProjectFile> language_programs,
            Vector<String> titles,
            Vector<String> objects,
            Vector<String> bodies,
            String flags
    ) {
        if (!language_programs.isEmpty()) {
            String LANG_ = language.toString().toUpperCase() + "_";
            Vector<String> module_objects = new Vector<>();
            String module_body = "";
            int i = 1;
            for (ProjectFile program : language_programs) {
                //--
                String object = Utils.DQuotes(language + "_" + i + ".o");
                module_objects.add(object);
                module_body +=
                        object + ":\n" +
                                "\t" +
                                String.join(" ",
                                        Utils.MFVar(LANG_ + "COMMAND"),
                                        Utils.MFVar(LANG_ + "FLAGS"),
                                        program.getStyleOptions(),
                                        "-c",
                                        program.getQSourceName(),
                                        "-o",
                                        object + "\n\n"
                                );
                ++i;
            }
            titles.add(String.join("\n",
                    LANG_ + "COMMAND=" + Utils.DQuotes(dvm_drv) + " " +
                            language.getDVMCompile(),
                    LANG_ + "FLAGS=" + flags,
                    LANG_ + "OBJECTS=" + String.join(" ", module_objects),
                    ""
            ));
            objects.add(Utils.MFVar(LANG_ + "OBJECTS"));
            bodies.add(module_body);
        }
    }
    static String generateMakefile(File test, LanguageName test_language, String dvm_drv, String flags) {
        //----->>
        LinkedHashMap<LanguageName, Vector<ProjectFile>> programs = getTestPrograms(test);
        Vector<String> titles = new Vector<>();
        Vector<String> objects = new Vector<>();
        Vector<String> bodies = new Vector<>();
        String binary = Utils.DQuotes("0");
        //----->>
        for (LanguageName languageName : programs.keySet()) {
            generateForLanguage(dvm_drv, languageName, programs.get(languageName), titles, objects, bodies, flags);
        }
        //----->>
        return String.join("\n",
                "LINK_COMMAND=" + Utils.DQuotes(dvm_drv) + " " +
                        test_language.getDVMLink(),
                "LINK_FLAGS=" + flags + "\n",
                String.join("\n", titles),
                "all: " + binary,
                binary + " : " + String.join(" ", objects),
                "\t" + Utils.MFVar("LINK_COMMAND") + " " + Utils.MFVar("LINK_FLAGS") + " " + String.join(" ", objects) + " -o " + binary,
                String.join(" ", bodies));
    }
    public void getTasksInfo(List<? extends DVMTask> tasks, String file_name) throws Exception {
        LinkedHashMap<Integer, DVMTask> sorted_tasks = new LinkedHashMap<>();
        for (DVMTask task : tasks)
            sorted_tasks.put(task.id, task);
        //--
        File info_file = Paths.get(packageLocalWorkspace.getAbsolutePath(), "results", file_name).toFile();
        List<String> lines = FileUtils.readLines(info_file, Charset.defaultCharset());
        for (String packed : lines) {
            if (!packed.isEmpty()) {
                String[] data = packed.split(" ");
                int id = Integer.parseInt(data[0]);
                TaskState state = TaskState.valueOf(data[1]);
                double time = Double.parseDouble(data[2]);
                //--
                DVMTask task = sorted_tasks.get(id);
                task.state = state;
                task.Time = state.equals(TaskState.AbortedByTimeout) ? (task.maxtime + 1) : time;
            }
        }
    }
    //---
    @Override
    protected ServerCode getActivePackageCode() {
        return ServerCode.GetFirstActiveDVMPackage;
    }
    @Override
    protected ServerCode getCheckIfNeedsKillCode() {
        return ServerCode.DVMPackageNeedsKill;
    }
    @Override
    protected TasksPackageState getStateAfterStart() {
        return TasksPackageState.CompilationWorkspacesCreation;
    }
    @Override
    protected void TestsSynchronize() throws Exception {
        testingPackage.readJson();
        LinkedHashMap<Integer, File> tests = getTestsFromJson();
        //синхронизировать их.
        for (int test_id : tests.keySet()) {
            // Print("testId="+test_id);
            File test = tests.get(test_id);
            RemoteFile test_dst = new RemoteFile(testingPackage.user_workspace + "/projects/" + test_id, true);
            //  Print("src="+test.getAbsolutePath());
            //  Print("dst="+test_dst.full_name);
            user.connection.MKDIR(test_dst);
            user.connection.SynchronizeSubDirsR(test, test_dst);
            //   Print("done");
        }
    }
    @Override
    protected void PackageWorkspaceCreation() throws Exception {
        testingPackage.readJson();
        //--
        LinkedHashMap<Integer, File> tests = getTestsFromJson();
        //создать папку для пакета.
        user.connection.sftpChannel.mkdir(packageRemoteWorkspace.full_name);
        //положить туда запакованные тексты задач.
        Vector<String> compilationLines = new Vector<>();
        Vector<String> runLines = new Vector<>();
        for (DVMCompilationTask compilationTask : testingPackage.package_json.compilationTasks) {
            String makefileText = generateMakefile(tests.get(compilationTask.test_id), compilationTask.language, testingPackage.drv, compilationTask.flags);
            compilationLines.addAll(compilationTask.pack(makefileText));
            for (DVMRunTask runTask : compilationTask.runTasks)
                runLines.addAll(runTask.pack(null));
        }
        RemoteFile compilationPackage = new RemoteFile(packageRemoteWorkspace, "compilationTasks");
        RemoteFile runPackage = new RemoteFile(packageRemoteWorkspace, "runTasks");
        user.connection.writeToFile(String.join("\n", compilationLines) + "\n", compilationPackage);
        user.connection.writeToFile(String.join("\n", runLines) + "\n", runPackage);
        // --
        user.connection.MKDIR(new RemoteFile(packageRemoteWorkspace, "state"));
    }
    @Override
    protected void PackageStart() throws Exception {
        user.connection.ShellCommand("ulimit -s unlimited"); // нужно, для запуска сишной части.
        String plannerStartCommand =
                String.join(" ",
                        "nohup",
                        Utils.DQuotes(getPlanner()),
                        Utils.DQuotes(user.workspace),
                        Utils.DQuotes(packageRemoteWorkspace.full_name),
                        Utils.DQuotes(testingPackage.kernels),
                        Utils.DQuotes(testingPackage.drv),
                        "&"
                );
        user.connection.ShellCommand(plannerStartCommand);
        RemoteFile PID = new RemoteFile(packageRemoteWorkspace, "PID");
        RemoteFile STARTED = new RemoteFile(packageRemoteWorkspace, "STARTED");
        while (!user.connection.Exists(STARTED)) {
            Print("waiting for package start...");
            Utils.sleep(1000);
        }
        if (user.connection.Exists(PID)) {
            testingPackage.PID = user.connection.readFromFile(PID);
        }
    }
    @Override
    protected boolean CheckNextState() throws Exception {

        boolean progress_changed = false;
        boolean state_changed = false;
        RemoteFile progress = new RemoteFile(packageRemoteWorkspace, "progress");
        if (user.connection.Exists(progress)) {
            String s = user.connection.readFromFile(progress);
            int current_progress = Integer.parseInt(s);
            if (current_progress != testingPackage.progress) {
                Print("progress changed: "+current_progress);
                testingPackage.progress = current_progress;
                progress_changed = true;
            }
        }
        RemoteFile stateDir = new RemoteFile(packageRemoteWorkspace, "state");
        //состояния пакета могут меняться только по возрастанию. ищем, появилось ли такое.
        Vector<TasksPackageState> higherStates = testingPackage.state.getHigherStates();
        Collections.reverse(higherStates); //берем в обратном порядке, чтобы быстрее найти высшее.
        for (TasksPackageState state : higherStates) {
            RemoteFile file = new RemoteFile(stateDir, state.toString());
            if (user.connection.Exists(file)) {
                Print("found new state: " + file.name);
                testingPackage.state = state;
                state_changed = true;
                break;
            }
        }
        //--
        user.connection.iterations++;
        if (user.connection.iterations==100) {
            Disconnect();
        }
        //--
        return progress_changed || state_changed;
    }
    @Override
    protected void DownloadResults() throws Exception {
        Utils.CheckDirectory(packageLocalWorkspace);
        RemoteFile remote_results_archive = new RemoteFile(packageRemoteWorkspace, "results.zip");
        File results_archive = new File(packageLocalWorkspace, "results.zip");
        user.connection.performScript(packageRemoteWorkspace, "zip -r " + Utils.DQuotes("results.zip") + " " + Utils.DQuotes("results"));
        //---
        if (user.connection.Exists(remote_results_archive)) {
            user.connection.getSingleFile(remote_results_archive.full_name, results_archive.getAbsolutePath());
            UnzipFolderPass unzipFolderPass = new UnzipFolderPass();
            unzipFolderPass.Do(results_archive.getAbsolutePath(), packageLocalWorkspace.getAbsolutePath(), false);
        }
        //todo привязать это к настройкам, чтобы можно было включать/выключать удаление.
        /*
        //получили результат. теперь уничтожаем папку пакета на целевой машине.
        if ( user.connection.Exists(packageRemoteWorkspace)){
            user.connection.RMDIR(packageRemoteWorkspace.full_name);
        }
         */
    }
    @Override
    protected void AnalyseResults() throws Exception {
        testingPackage.readJson();
        Print("analysing results");
        Vector<DVMRunTask> runTasks = new Vector<>();
        for (DVMCompilationTask compilationTask : testingPackage.package_json.compilationTasks)
            runTasks.addAll(compilationTask.runTasks);
        //----
        getTasksInfo(testingPackage.package_json.compilationTasks, "CompilationInfo.txt");
        getTasksInfo(runTasks, "RunningInfo.txt");
        //--
        int ct_count = 0;
        int rt_count = 0;
        //--
        for (DVMCompilationTask compilationTask : testingPackage.package_json.compilationTasks) {
            ct_count++;
            File ct_workspace = Paths.get(packageLocalWorkspace.getAbsolutePath(), "results", String.valueOf(compilationTask.id)).toFile();
            if (ct_workspace.exists()) {
                for (DVMRunTask runTask : compilationTask.runTasks) {
                    rt_count++;
                    runTask.compilation_state = compilationTask.state;
                    runTask.compilation_time = compilationTask.Time;
                    if (compilationTask.state == TaskState.DoneWithErrors) {
                        runTask.state = TaskState.Canceled;
                    } else {
                        File rt_workspace = Paths.get(packageLocalWorkspace.getAbsolutePath(), "results", String.valueOf(runTask.id)).toFile();
                        if (rt_workspace.exists() && runTask.state.equals(TaskState.Finished)) {
                            //анализ задачи на запуск.
                            File outFile = new File(rt_workspace, Constants.out_file);
                            File errFile = new File(rt_workspace.getAbsolutePath(), Constants.err_file);
                            //--
                            String output = FileUtils.readFileToString(outFile);
                            String errors = FileUtils.readFileToString(errFile);
                            //--
                            List<String> output_lines = Arrays.asList(output.split("\n"));
                            List<String> errors_lines = Arrays.asList(errors.split("\n"));
                            //---
                            if (Utils.isCrushed(output_lines, errors_lines)) {
                                runTask.state = TaskState.Crushed;
                            } else {
                                Pair<TaskState, Integer> results = new Pair<>(TaskState.Done, 100);
                                switch (runTask.test_type) {
                                    case Correctness:
                                        results = Utils.analyzeCorrectness(output_lines);
                                        break;
                                    case Performance:
                                        results = Utils.analyzePerformance(output_lines);
                                        break;
                                    default:
                                        break;
                                }
                                runTask.state = results.getKey();
                                runTask.progress = results.getValue();
                                runTask.CleanTime = Utils.parseCleanTime(output);
                            }
                        }
                    }
                }
            }
        }
        testingPackage.progress = 100;
        testingPackage.saveJson(); //запись обновленных результатов пакета в json!
        Print("analysis done, ct_count=" + ct_count + " rt count=" + rt_count);
    }
    @Override
    protected void Kill() throws Exception {
        if (!testingPackage.PID.isEmpty()) {
            user.connection.ShellCommand("kill -9 " + testingPackage.PID);
        }
    }
    @Override
    protected void InitSessionCredentials() {
        String machine_url = testingPackage.machine_address + ":" + testingPackage.machine_port;
        if (!machines.containsKey(machine_url))
            machines.put(machine_url, testingPackage.getMachine());
        if (!users.containsKey(testingPackage.user_name))
            users.put(testingPackage.user_name, testingPackage.getUser());
        //-->>
        machine = machines.get(machine_url);
        user = users.get(testingPackage.user_name);
        //--
        packageRemoteWorkspace = new RemoteFile(user.workspace + "/tests", String.valueOf(testingPackage.id), true);
        packageLocalWorkspace = new File(Global.PackagesDirectory, String.valueOf(testingPackage.id));
    }
    @Override
    protected boolean Connect() {
        if (user.connection==null) {
            try {
                user.connection = new UserConnection(machine, user);
                Print("Соединение c " + machine.getURL() + " " + user.login + " успешно установлено.");
            } catch (Exception ex) {
                Print(ex.toString());
                user.connection = null;
                Print("Не удалось установить соединение.");
            }
        }
        return user.connection != null;
    }
    @Override
    protected void Disconnect() {
        if (user.connection != null) {
            user.connection.Disconnect();
            user.connection = null;
        }
    }
    //--
}