Files
VisualSapfor/src/TestingSystem/TestingServer.java
2023-10-14 00:43:39 +03:00

558 lines
29 KiB
Java
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package TestingSystem;
import Common.Current;
import Common.Database.DBObject;
import Common.Database.rDBObject;
import Common.Global;
import Common.GlobalProperties;
import Common.Utils.Utils;
import GlobalData.Machine.Machine;
import GlobalData.RemoteFile.RemoteFile;
import GlobalData.Tasks.TaskState;
import GlobalData.User.User;
import ProjectData.LanguageName;
import ProjectData.Project.db_project_info;
import Repository.RepositoryRefuseException;
import Repository.RepositoryServer;
import Repository.Server.ServerCode;
import Repository.Server.ServerExchangeUnit_2021;
import SapforTestingSystem.Json.SapforConfiguration_json;
import SapforTestingSystem.Json.SapforTasksPackage_json;
import SapforTestingSystem.SapforConfiguration.SapforConfiguration;
import SapforTestingSystem.SapforConfiguration.SapforConfigurationInterface;
import SapforTestingSystem.SapforTasksPackage.SapforTasksPackage;
import SapforTestingSystem.SapforTasksPackage_info;
import SapforTestingSystem.ServerSapfor.ServerSapfor;
import TestingSystem.Group.Group;
import TestingSystem.Group.GroupInterface;
import TestingSystem.Tasks.TestCompilationTask;
import TestingSystem.Tasks.TestRunTask;
import TestingSystem.Tasks.TestTask;
import TestingSystem.TasksPackage.TasksPackage;
import TestingSystem.TasksPackage.TasksPackageState;
import TestingSystem.TasksPackageToKill.TasksPackageToKill;
import TestingSystem.Test.Test;
import TestingSystem.Test.TestInterface;
import TestingSystem.Test.TestType;
import Visual_DVM_2021.Passes.All.DownloadRepository;
import Visual_DVM_2021.Passes.All.UnzipFolderPass;
import Visual_DVM_2021.Passes.All.ZipFolderPass;
import Visual_DVM_2021.Passes.PassCode_2021;
import Visual_DVM_2021.Passes.Pass_2021;
import javafx.util.Pair;
import org.apache.commons.io.FileUtils;
import javax.swing.*;
import java.io.File;
import java.nio.file.Paths;
import java.util.Comparator;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.Vector;
import java.util.stream.Collectors;
import static TestingSystem.TasksDatabase.tests_db_name;
public class TestingServer extends RepositoryServer<TestsDatabase> {
LinkedHashMap<String, TasksDatabase> accountsBases = new LinkedHashMap<>();
//--------------------------------->>>
public TestingServer() {
super(TestsDatabase.class);
}
//основа
@Override
public int getPort() {
return 7998;
}
//---
public TasksDatabase account_db = null;
public void SetCurrentAccountDB(String email) {
if (accountsBases.containsKey(email)) {
account_db = accountsBases.get(email);
} else {
account_db = new TasksDatabase(email.equals("?") ? "undefined" : email);
accountsBases.put(email, account_db);
try {
account_db.Connect();
account_db.CreateAllTables();
account_db.prepareTablesStatements();
account_db.Synchronize();
} catch (Exception ex) {
ex.printStackTrace();
}
}
}
@Override
public void PublishAction(DBObject object) throws Exception {
if (object instanceof TasksPackage) {
//объект уже вставлен.
TasksPackage tasksPackage = (TasksPackage) object;
//-
for (String group_id : tasksPackage.sorted_tasks.keySet()) {
if (db.groups.containsKey(group_id)) {
Group group = db.groups.get(group_id);
LinkedHashMap<String, Vector<TestCompilationTask>> group_tasks = tasksPackage.sorted_tasks.get((group_id));
for (String test_id : group_tasks.keySet()) {
if (db.tests.containsKey(test_id)) {
Test test = db.tests.get(test_id);
db_project_info project = new db_project_info(test);//Открытие бд проекта и ее синхронизация. неизбежно.
//---
for (TestCompilationTask task : group_tasks.get(test_id)) {
Print("принять задачу на компиляцию " + group_id + ":" + test_id + ":" + task.flags);
//Теперь эту задачу надо поставить в очередь. и вернуть пользователю, уже с id
task.state = TaskState.Waiting;
task.id = db.IncMaxTaskId();
task.taskspackage_id = tasksPackage.id;
task.makefile_text = GroupInterface.GenerateMakefile(group, project, tasksPackage.dvm_drv, task.flags);
task.test_home = tasksPackage.user_workspace + "/projects/" + test_id;
//-->>
task.remote_workspace =
new RemoteFile(
tasksPackage.user_workspace + "/tests/" + tasksPackage.id,
String.valueOf(task.id), true).full_name;
account_db.Insert(task);
if (task.runTasks != null) {
for (TestRunTask rt : task.runTasks) {
rt.id = db.IncMaxTaskId();
rt.taskspackage_id = tasksPackage.id;
rt.testcompilationtask_id = task.id;
rt.remote_workspace =
new RemoteFile(
tasksPackage.user_workspace + "/tests/" + tasksPackage.id,
String.valueOf(rt.id), true).full_name;
rt.binary_name = "spf_" + rt.id + "_" + rt.matrix.replace(" ", "_");
account_db.Insert(rt);
}
}
}
}
}
}
}
} else if (object instanceof Test) {
Test new_test = (Test) object;
Utils.unpackFile(new_test.project_archive_bytes, TestInterface.getArchive(new_test));
//распаковать архив в папку с тестами. для тестирования удобнее хранить их уже открытыми.
UnzipFolderPass unzipFolderPass = new UnzipFolderPass();
if (!unzipFolderPass.Do(
TestInterface.getArchive(new_test).getAbsolutePath(),
TestInterface.getServerPath(new_test).getParentFile().getAbsolutePath()))
throw new RepositoryRefuseException("Не удалось распаковать Тест с именем " + new_test.id);
}
}
@Override
public void CopyAction(rDBObject src, rDBObject dst) throws Exception {
if (src instanceof Group) {
Group groupSrc = (Group) src;
Group groupDst = (Group) dst;
//тут есть право использовать Базу данных напрямую, ибо проход в серверной очереди.
//1. Получить список тестов.
Vector<Test> toCopy = db.tests.Data.values().stream().filter(test -> test.group_id.equals(groupSrc.id)).collect(Collectors.toCollection(Vector::new));
for (Test testSrc : toCopy) {
Test testCopy = new Test(testSrc);
testCopy.genName();
testCopy.sender_name = groupDst.sender_name;
testCopy.sender_address = groupDst.sender_address;
testCopy.group_id = groupDst.id;
//1 скопировать папку
FileUtils.copyDirectory(TestInterface.getServerPath(testSrc), TestInterface.getServerPath(testCopy));
//создать архив из скопированной папки.
ZipFolderPass zip = new ZipFolderPass();
if (!zip.Do(TestInterface.getServerPath(testCopy).getAbsolutePath(), TestInterface.getArchive(testCopy).getAbsolutePath()))
throw new RepositoryRefuseException("Не удалось создать архив копии теста");
db.Insert(testCopy);
}
}
}
@Override
public void DeleteAction(DBObject object) throws Exception {
if (object instanceof Test) {
Test test = (Test) object;
Utils.forceDeleteWithCheck(TestInterface.getArchive(test));
Utils.forceDeleteWithCheck(TestInterface.getServerPath(test));
} else if (object instanceof Group) {
Group group = (Group) object;
Vector<Test> tests = new Vector<>();
for (Test group_test : db.tests.Data.values()) {
if (group_test.group_id.equals(group.id)) // todo group_name -> group_id
tests.add(group_test);
}
for (Test group_test : tests) {
db.Delete(group_test);
Utils.forceDeleteWithCheck(TestInterface.getArchive(group_test));
Utils.forceDeleteWithCheck(TestInterface.getServerPath(group_test));
}
} else if (object instanceof ServerSapfor) {
Utils.forceDeleteWithCheck(
new File(
((ServerSapfor) object).home_path
)
);
} else if (object instanceof SapforTasksPackage) {
Utils.forceDeleteWithCheck(
new File(
((SapforTasksPackage) object).workspace
)
);
}
}
@Override
public boolean canDelete(DBObject object) throws Exception {
if (object instanceof TestTask) {
return !((TestTask) object).state.equals(TaskState.Running);
} else
return super.canDelete(object);
}
public void TestsSynchronize(String userWorkspace, Vector<Object> args) throws Exception {
Machine machine = (Machine) args.get(0);
User user = (User) args.get(1);
Vector<String> test_ids = (Vector<String>) args.get(2);
//---->>>
UserConnection connection = new UserConnection(machine, user);
for (String test_id : test_ids) {
File test_src = Paths.get(Global.TestsDirectory.getAbsolutePath(), test_id).toFile();
RemoteFile test_dst = new RemoteFile(userWorkspace + "/projects/" + test_id, true);
connection.MKDIR(test_dst);
connection.SynchronizeSubDirsR(test_src, test_dst);
}
//---->>>
connection.Disconnect();
}
//--->>
@Override
protected void startAdditionalThreads() {
testingThread.start();
}
protected TestingPlanner testingPlanner = new TestingPlanner();
protected Thread testingThread = new Thread(() -> testingPlanner.Perform());
/*
protected ServerSapforTestingPlanner sapforTestingPlanner = new ServerSapforTestingPlanner();
protected Thread sapforTestingThread = new Thread(() -> sapforTestingPlanner.Perform());
*/
//------>>>
public static boolean checkTasks = false;
//--
public static void switchTimer(boolean on) {
if (on)
TimerOn();
else
TimerOff();
}
//----
public static int checkIntervalSecond = 10;
public static int kernels = 4;
//----
public static Timer checkTimer = null;
public static void TimerOn() {
checkTasks = true;
checkTimer = new Timer(checkIntervalSecond * 1000, e -> {
Pass_2021.passes.get(PassCode_2021.ActualizePackages).Do();
});
checkTimer.start();
}
public static void TimerOff() {
System.out.println("timer off");
if (checkTimer != null)
checkTimer.stop();
checkTasks = false;
}
public static void ResetTimer() {
TimerOff();
TimerOn();
}
//->>
Group ConvertDirectoryToGroup(File src, LanguageName languageName, TestType testType) throws Exception {
Group object = new Group();
//->>
object.description = src.getName();
object.language = languageName;
object.type = testType;
//-->>
//->>
File[] testsFiles = src.listFiles(pathname ->
pathname.isFile()
&& !pathname.getName().equals("settings")
&& !pathname.getName().equals("test-analyzer.sh")
&& Utils.getExtension(pathname).startsWith(languageName.getDVMCompile()));
;
if (testsFiles != null) {
for (File testFile : testsFiles)
object.testsFiles.put(testFile.getName(), Utils.packFile(testFile));
}
//->>
return object;
}
//->>
public Vector<Group> getRepoGroupsInfo() throws Exception {
Vector<Group> groups = new Vector<>();
File testsSrc = Paths.get(
Global.RepoDirectory.getAbsolutePath(),
"dvm", "tools", "tester", "trunk", "test-suite").toFile();
LanguageName[] supportedLanguages = new LanguageName[]{LanguageName.fortran, LanguageName.c};
for (LanguageName languageName : supportedLanguages) {
for (TestType testType : TestType.values()) {
File groupsSrc = null;
switch (testType) {
case Correctness:
String languageSrcName = null;
switch (languageName) {
case fortran:
languageSrcName = "Fortran";
break;
case c:
languageSrcName = "C";
break;
}
if (languageSrcName != null) {
groupsSrc = Paths.get(testsSrc.getAbsolutePath(), "Correctness", languageSrcName).toFile();
File[] groupsDirs = groupsSrc.listFiles(File::isDirectory);
if (groupsDirs != null) {
for (File groupDir : groupsDirs)
groups.add(ConvertDirectoryToGroup(groupDir, languageName, testType));
}
}
break;
case Performance:
File groupDir = Paths.get(testsSrc.getAbsolutePath(), "Performance").toFile();
groups.add(ConvertDirectoryToGroup(groupDir, languageName, testType));
break;
}
}
}
groups.sort(Comparator.comparing(o -> o.description));
return groups;
}
public void StartSapforTests(SapforTasksPackage_info sapforTasksPackage_info) throws Exception {
SetCurrentAccountDB(sapforTasksPackage_info.email);
//--
SapforTasksPackage_json package_json = new SapforTasksPackage_json();
for (String test_id : sapforTasksPackage_info.testsIds)
package_json.tests.add(db.tests.get(test_id).description);
//создание объекта набора задач. для бд.
SapforTasksPackage sapforTasksPackage = new SapforTasksPackage(); //?может быть все же поменять ключ на сгенерированный из даты.
sapforTasksPackage.genName();
account_db.Insert(sapforTasksPackage);
//создание рабочего пространства для пакетного режима
File packageWorkspace = new File(Global.SapforPackagesDirectory, String.valueOf(sapforTasksPackage.id));
Utils.CheckAndCleanDirectory(packageWorkspace);
sapforTasksPackage.workspace = packageWorkspace.getAbsolutePath();
//копирование тестов по конфигурациям.
for (String configuration_id : sapforTasksPackage_info.configurationsIds) {
SapforConfiguration configuration = db.sapforConfigurations.get(configuration_id);
//--
SapforConfiguration_json configuration_json = new SapforConfiguration_json();
configuration_json.id = String.valueOf(configuration_id);
configuration_json.flags = SapforConfigurationInterface.getFlags(configuration);
configuration_json.codes.addAll(SapforConfigurationInterface.getPassCodes(configuration));
//--->>
package_json.configurations.add(configuration_json);
//-->>
File configurationWorkspace = new File(packageWorkspace, configuration_id);
FileUtils.forceMkdir(configurationWorkspace);
for (String test_id : sapforTasksPackage_info.testsIds) {
Test test = db.tests.get(test_id);
File test_root = new File(configurationWorkspace, test.description);
Utils.CheckAndCleanDirectory(test_root);
FileUtils.copyDirectory(new File(Global.TestsDirectory, test.id), test_root);
sapforTasksPackage.tasksCount++;
}
}
//копирование SAPFOR
File sapforFile = new File(sapforTasksPackage.workspace, Utils.getDateName("SAPFOR_F"));
ServerSapfor sapfor = db.serverSapfors.get(sapforTasksPackage_info.sapforId);
FileUtils.copyFile(new File(sapfor.call_command), sapforFile);
if (!sapforFile.setExecutable(true))
throw new RepositoryRefuseException("Не удалось сделать файл " + sapforFile.getName() + " исполняемым!");
package_json.sapfor_drv = sapforFile.getName();
//--->>
//копирование визуализатора
File visualiser = new File(sapforTasksPackage.workspace, "VisualSapfor.jar");
FileUtils.copyFile(new File(Global.Home, "TestingSystem.jar"), visualiser);
//создание настроек
GlobalProperties properties = new GlobalProperties();
properties.Mode = Current.Mode.Package;
properties.threadsNum = Global.properties.threadsNum; //брать из настроек сервера же.
properties.threadsTimeout = Global.properties.threadsTimeout;
Utils.jsonToFile(properties, new File(sapforTasksPackage.workspace, "properties"));
//создание инструкции
File package_json_file = new File(sapforTasksPackage.workspace, "package_json");
Utils.jsonToFile(package_json, package_json_file);
//подготовка пакетного режима. Запустит его уже очередь.
Utils.createScript(packageWorkspace, packageWorkspace, "start", "java -jar VisualSapfor.jar");
//--
sapforTasksPackage.sapfor_version = sapfor.version;
sapforTasksPackage.sapfor_build_date = sapfor.buildDate;
sapforTasksPackage.testsNames = String.join(";", package_json.tests);
sapforTasksPackage.StartDate = new Date().getTime();
sapforTasksPackage.state = TasksPackageState.Queued;
account_db.Update(sapforTasksPackage);
}
@Override
protected void Session() throws Exception {
DBObject dbObject = null;
SapforTasksPackage_info scenario = null;
Test test = null;
switch (code) {
case StartSapforTests:
Print("Запуск тестирования SAPFOR");
StartSapforTests((SapforTasksPackage_info) request.object);
response = new ServerExchangeUnit_2021(ServerCode.OK);
break;
case SynchronizeTests:
//временный проход. синхронизирует тесты на заданной машине, с сервера.
Print("Синхронизация тестов");
TestsSynchronize(request.arg, (Vector<Object>) request.object);
//------------->>
response = new ServerExchangeUnit_2021(ServerCode.OK);
break;
//--
case CheckPackageToKill:
SetCurrentAccountDB(request.arg);
String packageName = (String) request.object;
response = new ServerExchangeUnit_2021(ServerCode.OK);
boolean res_ = false;
for (TasksPackageToKill tasksPackageToKill : account_db.packagesToKill.Data.values()) {
if (tasksPackageToKill.packageName.equals(packageName)) {
res_ = true;
break;
}
}
response.object = res_;
break;
case EditAccountObject:
SetCurrentAccountDB(request.arg);
DBObject new_object = (DBObject) request.object;
Print("Редактировать объект " + new_object.getPK() + " для пользователя " + request.arg);
account_db.UpdateWithCheck(new_object);
response = new ServerExchangeUnit_2021(ServerCode.OK);
break;
case GetAccountObjectCopyByPK:
SetCurrentAccountDB(request.arg);
Pair<Class, Object> p = (Pair<Class, Object>) request.object;
Print("Получить для пользователя " + request.arg + " копию объекта класса " + p.getKey().toString() + " по ключу " + p.getValue());
dbObject = account_db.getObjectCopyByPK(p.getKey(), p.getValue());
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = dbObject;
break;
case DeleteAccountObjects:
Print("Удалить список объектов с базы пользователя " + request.arg);
SetCurrentAccountDB(request.arg);
Vector<Object> objects = (Vector<Object>) request.object;
account_db.BeginTransaction();
for (Object object : objects) {
dbObject = (DBObject) object;
if (canDelete(dbObject)) {
account_db.DeleteWithCheck(dbObject);
DeleteAction(dbObject);
}
}
account_db.Commit();
response = new ServerExchangeUnit_2021(ServerCode.OK);
break;
case PublishAccountObjects:
Print("Опубликовать объекты для пользователя " + request.arg);
SetCurrentAccountDB(request.arg);
Vector<Object> objects_ = (Vector<Object>) request.object;
account_db.BeginTransaction();
for (Object object : objects_)
if (account_db.InsertWithCheck_((DBObject) object) != null)
PublishAction((DBObject) object);
account_db.Commit();
response = new ServerExchangeUnit_2021(ServerCode.OK);
break;
case CheckAccountObjectExistense:
SetCurrentAccountDB(request.arg);
p = (Pair<Class, Object>) request.object;
Print("Проверить существование объекта класса для пользователя " + request.arg + " " + p.getKey().toString() + " с ключом " + p.getValue());
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = account_db.checkObjectExistense(p.getKey(), p.getValue());
break;
//------------------------------------------->>
case DownloadTest:
Print("Отправить клиенту тест " + request.arg);
if (db.tests.containsKey(request.arg)) {
test = db.tests.get(request.arg);
response = new ServerExchangeUnit_2021(ServerCode.OK, "", Utils.packFile(TestInterface.getArchive(test)));
} else
throw new RepositoryRefuseException("Теста с именем " + request.arg + " не существует");
break;
//-------------------------------------------------------------------------------------->>>>
case RefreshDVMTests:
Print("Синхронизировать репозиторий тестов");
// временно отключить для отладки.
DownloadRepository downloadRepository = new DownloadRepository();
if (!downloadRepository.Do())
throw new RepositoryRefuseException("Не удалось обновить репозиторий");
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = getRepoGroupsInfo();
break;
case GetAccountQueueSize:
Print("Получить размер очереди для пользователя " + request.arg);
SetCurrentAccountDB(request.arg);
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = (int) account_db.testCompilationTasks.Data.values().stream().filter(ctask -> ctask.state.isActive()).count()
+ (int) account_db.testRunTasks.Data.values().stream().filter(rtask -> rtask.state.isActive()).count();
break;
case GetAccountObjectsCopiesByPKs:
Print("Получить список копий объектов для пользователя " + request.arg);
p = (Pair<Class, Object>) request.object;
SetCurrentAccountDB(request.arg);
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = account_db.getObjectsCopies(p.getKey(), (Vector<Object>) p.getValue());
break;
case GetFirstActiveAccountPackage:
Print("Получить первый активный пакет задач для пользователя " + request.arg);
SetCurrentAccountDB(request.arg);
response = new ServerExchangeUnit_2021(ServerCode.OK);
TasksPackage tasksPackage = account_db.getFirstActivePackage();
LinkedHashMap<Long, TestCompilationTask> activeTasks = account_db.getPackageCompilationTasks(tasksPackage);
response.object = new Pair<>(tasksPackage, activeTasks);
break;
case GetFirstActiveSapforTasksPackage:
Print("Получить первый активный сценарий задач SAPFOR" + request.arg);
SetCurrentAccountDB(request.arg);
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = account_db.getFirstActiveSapforScenario();
break;
case GetQueueSize:
Print("Получить размер глобальной очереди задач");
long date = (long) request.object;
long res = 0;
Vector<String> emails = new Vector<>();
File[] accountsBases_ = Global.DataDirectory.listFiles(pathname ->
pathname.isFile() &&
Utils.getExtension(pathname).equals("sqlite") &&
!Utils.getNameWithoutExtension(pathname.getName()).isEmpty() &&
!pathname.getName().equals(tests_db_name + ".sqlite")
);
if (accountsBases_ != null) {
for (File accountBase : accountsBases_) {
String fileName = accountBase.getName();
String account_email = accountBase.getName().substring(0, fileName.lastIndexOf('_'));
emails.add(account_email);
}
for (String email : emails) {
SetCurrentAccountDB(email);
res += account_db.getQueueSize(date);
}
}
//пройтись по всем аккаунтам, и узнать все пакеты, чья дата меньше равна дате в арге
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = res;
break;
case ReceiveTestsDatabase:
Print("Получить базу данных тестов");
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = Utils.packFile(db.getFile());
break;
case ReceiveTestsTasksDatabase:
Print("Получить базу данных тестовых задач пользователя " + request.arg);
SetCurrentAccountDB(request.arg);
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = Utils.packFile(account_db.getFile());
break;
default:
throw new RepositoryRefuseException("Неподдерживаемый код: " + code);
}
}
}