Files
VisualSapfor/src/TestingSystem/TestingServer.java
2023-11-08 23:38:07 +03:00

567 lines
30 KiB
Java
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package TestingSystem;
import Common.Constants;
import Common.Database.DBObject;
import Common.Database.rDBObject;
import Common.Global;
import Common.Utils.Utils;
import GlobalData.Machine.Machine;
import GlobalData.RemoteFile.RemoteFile;
import GlobalData.Tasks.TaskState;
import GlobalData.User.User;
import ProjectData.LanguageName;
import ProjectData.Project.db_project_info;
import Repository.RepositoryRefuseException;
import Repository.RepositoryServer;
import Repository.Server.ServerCode;
import Repository.Server.ServerExchangeUnit_2021;
import SapforTestingSystem.Json.SapforTasksResults_json;
import SapforTestingSystem.SapforTask.SapforTask;
import SapforTestingSystem.SapforTasksPackage.SapforTasksPackage;
import SapforTestingSystem.ServerSapfor.ServerSapfor;
import TestingSystem.Group.Group;
import TestingSystem.Group.GroupInterface;
import TestingSystem.Tasks.TestCompilationTask;
import TestingSystem.Tasks.TestRunTask;
import TestingSystem.Tasks.TestTask;
import TestingSystem.TasksPackage.TasksPackage;
import TestingSystem.TasksPackageToKill.TasksPackageToKill;
import TestingSystem.Test.Test;
import TestingSystem.Test.TestInterface;
import TestingSystem.Test.TestType;
import Visual_DVM_2021.Passes.All.DownloadRepository;
import Visual_DVM_2021.Passes.All.UnzipFolderPass;
import Visual_DVM_2021.Passes.All.ZipFolderPass;
import Visual_DVM_2021.Passes.PassCode_2021;
import Visual_DVM_2021.Passes.Pass_2021;
import javafx.util.Pair;
import org.apache.commons.io.FileUtils;
import javax.swing.*;
import java.io.File;
import java.nio.file.Paths;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.Vector;
import java.util.stream.Collectors;
import static TestingSystem.TasksDatabase.tests_db_name;
public class TestingServer extends RepositoryServer<TestsDatabase> {
public static boolean email = false;
public static boolean checkTasks = false;
public static int checkIntervalSecond = 10;
public static int kernels = 4;
LinkedHashMap<String, TasksDatabase> accountsBases = new LinkedHashMap<>();
//--------------------------------->>>
public TestingServer() {
super(TestsDatabase.class);
}
//основа
@Override
public int getPort() {
return 7998;
}
//---
public TasksDatabase account_db = null;
public void SetCurrentAccountDB(String email) {
if (accountsBases.containsKey(email)) {
account_db = accountsBases.get(email);
} else {
account_db = new TasksDatabase(email.equals("?") ? "undefined" : email);
accountsBases.put(email, account_db);
try {
account_db.Connect();
account_db.CreateAllTables();
account_db.prepareTablesStatements();
account_db.Synchronize();
} catch (Exception ex) {
ex.printStackTrace();
}
}
}
@Override
public void PublishAction(DBObject object) throws Exception {
if (object instanceof TasksPackage) {
//объект уже вставлен.
TasksPackage tasksPackage = (TasksPackage) object;
//-
for (String group_id : tasksPackage.sorted_tasks.keySet()) {
if (db.groups.containsKey(group_id)) {
Group group = db.groups.get(group_id);
LinkedHashMap<String, Vector<TestCompilationTask>> group_tasks = tasksPackage.sorted_tasks.get((group_id));
for (String test_id : group_tasks.keySet()) {
if (db.tests.containsKey(test_id)) {
Test test = db.tests.get(test_id);
db_project_info project = new db_project_info(test);//Открытие бд проекта и ее синхронизация. неизбежно.
//---
for (TestCompilationTask task : group_tasks.get(test_id)) {
Print("принять задачу на компиляцию " + group_id + ":" + test_id + ":" + task.flags);
//Теперь эту задачу надо поставить в очередь. и вернуть пользователю, уже с id
task.state = TaskState.Waiting;
task.id = db.IncMaxTaskId();
task.taskspackage_id = tasksPackage.id;
task.makefile_text = GroupInterface.GenerateMakefile(group, project, tasksPackage.dvm_drv, task.flags);
task.test_home = tasksPackage.user_workspace + "/projects/" + test_id;
//-->>
task.remote_workspace =
new RemoteFile(
tasksPackage.user_workspace + "/tests/" + tasksPackage.id,
String.valueOf(task.id), true).full_name;
account_db.Insert(task);
if (task.runTasks != null) {
for (TestRunTask rt : task.runTasks) {
rt.id = db.IncMaxTaskId();
rt.taskspackage_id = tasksPackage.id;
rt.testcompilationtask_id = task.id;
rt.remote_workspace =
new RemoteFile(
tasksPackage.user_workspace + "/tests/" + tasksPackage.id,
String.valueOf(rt.id), true).full_name;
rt.binary_name = "spf_" + rt.id + "_" + rt.matrix.replace(" ", "_");
account_db.Insert(rt);
}
}
}
}
}
}
}
} else if (object instanceof Test) {
Test new_test = (Test) object;
Utils.unpackFile(new_test.project_archive_bytes, TestInterface.getArchive(new_test));
//распаковать архив в папку с тестами. для тестирования удобнее хранить их уже открытыми.
UnzipFolderPass unzipFolderPass = new UnzipFolderPass();
if (!unzipFolderPass.Do(
TestInterface.getArchive(new_test).getAbsolutePath(),
TestInterface.getServerPath(new_test).getParentFile().getAbsolutePath()))
throw new RepositoryRefuseException("Не удалось распаковать Тест с именем " + new_test.id);
}
}
@Override
public void CopyAction(rDBObject src, rDBObject dst) throws Exception {
if (src instanceof Group) {
Group groupSrc = (Group) src;
Group groupDst = (Group) dst;
//тут есть право использовать Базу данных напрямую, ибо проход в серверной очереди.
//1. Получить список тестов.
Vector<Test> toCopy = db.tests.Data.values().stream().filter(test -> test.group_id.equals(groupSrc.id)).collect(Collectors.toCollection(Vector::new));
for (Test testSrc : toCopy) {
Test testCopy = new Test(testSrc);
testCopy.genName();
testCopy.sender_name = groupDst.sender_name;
testCopy.sender_address = groupDst.sender_address;
testCopy.group_id = groupDst.id;
//1 скопировать папку
FileUtils.copyDirectory(TestInterface.getServerPath(testSrc), TestInterface.getServerPath(testCopy));
//создать архив из скопированной папки.
ZipFolderPass zip = new ZipFolderPass();
if (!zip.Do(TestInterface.getServerPath(testCopy).getAbsolutePath(), TestInterface.getArchive(testCopy).getAbsolutePath()))
throw new RepositoryRefuseException("Не удалось создать архив копии теста");
db.Insert(testCopy);
}
}
}
@Override
public void DeleteAction(DBObject object) throws Exception {
if (object instanceof Test) {
Test test = (Test) object;
Utils.forceDeleteWithCheck(TestInterface.getArchive(test));
Utils.forceDeleteWithCheck(TestInterface.getServerPath(test));
} else if (object instanceof Group) {
Group group = (Group) object;
Vector<Test> tests = new Vector<>();
for (Test group_test : db.tests.Data.values()) {
if (group_test.group_id.equals(group.id)) // todo group_name -> group_id
tests.add(group_test);
}
for (Test group_test : tests) {
db.Delete(group_test);
Utils.forceDeleteWithCheck(TestInterface.getArchive(group_test));
Utils.forceDeleteWithCheck(TestInterface.getServerPath(group_test));
}
} else if (object instanceof ServerSapfor) {
Utils.forceDeleteWithCheck(
new File(
((ServerSapfor) object).home_path
)
);
} else if (object instanceof SapforTasksPackage) {
SapforTasksPackage sapforTasksPackage = (SapforTasksPackage) object;
File workspace = new File(
sapforTasksPackage.workspace
);
System.out.println(Utils.Brackets(workspace.getAbsolutePath()));
Utils.forceDeleteWithCheck(workspace);
Utils.forceDeleteWithCheck(sapforTasksPackage.getArchive());
//внешние ключи не работают
Vector<SapforTask> tasks = new Vector<>();
for (SapforTask task : account_db.sapforTasks.Data.values()) {
if (task.sapfortaskspackage_id.equals(sapforTasksPackage.id)) // todo group_name -> group_id
tasks.add(task);
}
for (SapforTask task : tasks) {
account_db.Delete(task);
}
}
}
@Override
public boolean canDelete(DBObject object) throws Exception {
if (object instanceof TestTask) {
return !((TestTask) object).state.equals(TaskState.Running);
} else
return super.canDelete(object);
}
public void TestsSynchronize(String userWorkspace, Vector<Object> args) throws Exception {
Machine machine = (Machine) args.get(0);
User user = (User) args.get(1);
Vector<String> test_ids = (Vector<String>) args.get(2);
//---->>>
UserConnection connection = new UserConnection(machine, user);
for (String test_id : test_ids) {
File test_src = Paths.get(Global.TestsDirectory.getAbsolutePath(), test_id).toFile();
RemoteFile test_dst = new RemoteFile(userWorkspace + "/projects/" + test_id, true);
connection.MKDIR(test_dst);
connection.SynchronizeSubDirsR(test_src, test_dst);
}
//---->>>
connection.Disconnect();
}
//--->>
@Override
protected void startAdditionalThreads() {
testingThread.start();
}
protected TestingPlanner testingPlanner = new TestingPlanner();
protected Thread testingThread = new Thread(() -> testingPlanner.Perform());
//------>>>
public static void switchTimer(boolean on) {
if (on)
TimerOn();
else
TimerOff();
}
public static Timer checkTimer = null;
public static void TimerOn() {
checkTasks = true;
checkTimer = new Timer(checkIntervalSecond * 1000, e -> {
// Pass_2021.passes.get(PassCode_2021.ActualizePackages).Do();
Pass_2021.passes.get(PassCode_2021.SynchronizeTestsTasks).Do();
});
checkTimer.start();
}
public static void TimerOff() {
System.out.println("timer off");
if (checkTimer != null)
checkTimer.stop();
checkTasks = false;
}
public static void ResetTimer() {
TimerOff();
TimerOn();
}
//->>
Group ConvertDirectoryToGroup(File src, LanguageName languageName, TestType testType) throws Exception {
Group object = new Group();
//->>
object.description = src.getName();
object.language = languageName;
object.type = testType;
//-->>
//->>
File[] testsFiles = src.listFiles(pathname ->
pathname.isFile()
&& !pathname.getName().equals("settings")
&& !pathname.getName().equals("test-analyzer.sh")
&& Utils.getExtension(pathname).startsWith(languageName.getDVMCompile()));
;
if (testsFiles != null) {
for (File testFile : testsFiles)
object.testsFiles.put(testFile.getName(), Utils.packFile(testFile));
}
//->>
return object;
}
//->>
public Vector<Group> getRepoGroupsInfo() throws Exception {
Vector<Group> groups = new Vector<>();
File testsSrc = Paths.get(
Global.RepoDirectory.getAbsolutePath(),
"dvm", "tools", "tester", "trunk", "test-suite").toFile();
LanguageName[] supportedLanguages = new LanguageName[]{LanguageName.fortran, LanguageName.c};
for (LanguageName languageName : supportedLanguages) {
for (TestType testType : TestType.values()) {
File groupsSrc = null;
switch (testType) {
case Correctness:
String languageSrcName = null;
switch (languageName) {
case fortran:
languageSrcName = "Fortran";
break;
case c:
languageSrcName = "C";
break;
}
if (languageSrcName != null) {
groupsSrc = Paths.get(testsSrc.getAbsolutePath(), "Correctness", languageSrcName).toFile();
File[] groupsDirs = groupsSrc.listFiles(File::isDirectory);
if (groupsDirs != null) {
for (File groupDir : groupsDirs)
groups.add(ConvertDirectoryToGroup(groupDir, languageName, testType));
}
}
break;
case Performance:
File groupDir = Paths.get(testsSrc.getAbsolutePath(), "Performance").toFile();
groups.add(ConvertDirectoryToGroup(groupDir, languageName, testType));
break;
}
}
}
groups.sort(Comparator.comparing(o -> o.description));
return groups;
}
public SapforTasksResults_json getSapforPackageResults(SapforTasksPackage sapforTasksPackage) throws Exception {
File results_file = new File(sapforTasksPackage.workspace, Constants.results_json);
return (SapforTasksResults_json) Utils.jsonFromFile(
results_file, SapforTasksResults_json.class);
}
//надо в другое место.
/*
public Vector<String> CompareSapforPackages(SapforTasksPackage package1, SapforTasksPackage package2) throws Exception {
Vector<String> comparisonLog = new Vector<>();
// чтобы сравнивать. должен быть идентичный список тестов. (без учета последовательности)
// идентичный набор проходов (с учетом последовательности)
//идентичный набор флагов (?) без учета последовательности.
// то есть должны отличаться только тестируемые версии САПФОР.
SapforTasksResults_json results1_json = getSapforPackageResults(package1);
SapforTasksResults_json results2_json = getSapforPackageResults(package2);
if (results1_json.tasks.size() != results2_json.tasks.size()) {
comparisonLog.add("Количество задач в пакетах не совпадает.");
comparisonLog.add(package1.id + ": " + results1_json.tasks.size() + "/" + package2.id + ": " + results2_json.tasks.size());
return comparisonLog;
}
//подходит если все задачи из первого пакета содержатся во втором, и размер одинаковый.
Vector<SapforTask> sortedTasks2 = new Vector<>();
for (SapforTask task1 : results1_json.tasks) {
//на
boolean match = false;
for (SapforTask task2 : results2_json.tasks) {
if (task1.isComparable(task2)) {
match = true;
sortedTasks2.add(task2);
break;
}
}
if (!match) {
comparisonLog.add("Найдена не совпадающая задача.");
return comparisonLog;
}
}
for (SapforTask task1 : results1_json.tasks) {
for (SapforTask task2 : sortedTasks2) {
task1.Compare(task2, comparisonLog);
}
}
return comparisonLog;
}
*/
@Override
protected void Session() throws Exception {
DBObject dbObject = null;
Test test = null;
switch (code) {
case PublishSapforPackageTasks:
Print("Опубликовать задачи SAPFOR для пользователя " + request.arg);
SetCurrentAccountDB(request.arg);
Vector<Object> tasks = (Vector<Object>) request.object;
account_db.BeginTransaction();
for (Object object : tasks) {
SapforTask task = (SapforTask) object;
task.id = db.IncMaxTaskId();
if (account_db.InsertWithCheck_(task) != null)
PublishAction(task);
}
account_db.Commit();
response = new ServerExchangeUnit_2021(ServerCode.OK);
break;
case DownloadSapforTasksPackage:
Print("Загрузить пакет тестов SAPFOR " + request.object + " для пользователя " + request.arg);
SetCurrentAccountDB(request.arg);
response = new ServerExchangeUnit_2021(ServerCode.OK);
//---
if (!account_db.sapforTasksPackages.containsKey(request.object))
throw new RepositoryRefuseException("Не существует пакета с ключом " + Utils.Brackets(request.object));
//--
SapforTasksPackage sapforTasksPackage = account_db.sapforTasksPackages.get(request.object);
//---
//1 - архивировать пакет.
File packageArchive = sapforTasksPackage.getArchive();
Utils.forceDeleteWithCheck(packageArchive);
System.out.println("src = "+Utils.Brackets(sapforTasksPackage.workspace));
System.out.println("dst="+Utils.Brackets(packageArchive.getAbsolutePath()));
//---
ZipFolderPass zip = new ZipFolderPass();
if (zip.Do(sapforTasksPackage.workspace, packageArchive.getAbsolutePath())) {
response.object = Utils.packFile(packageArchive);
Print("Архив успешно запакован");
} else throw new RepositoryRefuseException("Не удалось запаковать архив пакета");
//---
break;
case SynchronizeTests:
//временный проход. синхронизирует тесты на заданной машине, с сервера.
Print("Синхронизация тестов");
TestsSynchronize(request.arg, (Vector<Object>) request.object);
//------------->>
response = new ServerExchangeUnit_2021(ServerCode.OK);
break;
case CheckPackageToKill:
SetCurrentAccountDB(request.arg);
String packageName = (String) request.object;
response = new ServerExchangeUnit_2021(ServerCode.OK);
boolean res_ = false;
for (TasksPackageToKill tasksPackageToKill : account_db.packagesToKill.Data.values()) {
if (tasksPackageToKill.packageName.equals(packageName)) {
res_ = true;
break;
}
}
response.object = res_;
break;
case EditAccountObject:
SetCurrentAccountDB(request.arg);
DBObject new_object = (DBObject) request.object;
Print("Редактировать объект " + new_object.getPK() + " для пользователя " + request.arg);
account_db.UpdateWithCheck(new_object);
response = new ServerExchangeUnit_2021(ServerCode.OK);
break;
case GetAccountObjectCopyByPK:
SetCurrentAccountDB(request.arg);
Pair<Class, Object> p = (Pair<Class, Object>) request.object;
Print("Получить для пользователя " + request.arg + " копию объекта класса " + p.getKey().toString() + " по ключу " + p.getValue());
dbObject = account_db.getObjectCopyByPK(p.getKey(), p.getValue());
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = dbObject;
break;
case DeleteAccountObjects:
Print("Удалить список объектов с базы пользователя " + request.arg);
SetCurrentAccountDB(request.arg);
Vector<Object> objects = (Vector<Object>) request.object;
account_db.BeginTransaction();
for (Object object : objects) {
dbObject = (DBObject) object;
if (canDelete(dbObject)) {
account_db.DeleteWithCheck(dbObject);
DeleteAction(dbObject);
}
}
account_db.Commit();
response = new ServerExchangeUnit_2021(ServerCode.OK);
break;
case PublishAccountObjects:
Print("Опубликовать объекты для пользователя " + request.arg);
SetCurrentAccountDB(request.arg);
Vector<Object> objects__ = (Vector<Object>) request.object;
account_db.BeginTransaction();
for (Object object : objects__)
if (account_db.InsertWithCheck_((DBObject) object) != null)
PublishAction((DBObject) object);
account_db.Commit();
response = new ServerExchangeUnit_2021(ServerCode.OK);
break;
case CheckAccountObjectExistense:
SetCurrentAccountDB(request.arg);
p = (Pair<Class, Object>) request.object;
Print("Проверить существование объекта класса для пользователя " + request.arg + " " + p.getKey().toString() + " с ключом " + p.getValue());
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = account_db.checkObjectExistense(p.getKey(), p.getValue());
break;
//------------------------------------------->>
case DownloadTest:
Print("Отправить клиенту тест " + request.arg);
if (db.tests.containsKey(request.arg)) {
test = db.tests.get(request.arg);
response = new ServerExchangeUnit_2021(ServerCode.OK, "", Utils.packFile(TestInterface.getArchive(test)));
} else
throw new RepositoryRefuseException("Теста с именем " + request.arg + " не существует");
break;
//-------------------------------------------------------------------------------------->>>>
case RefreshDVMTests:
Print("Синхронизировать репозиторий тестов");
// временно отключить для отладки.
DownloadRepository downloadRepository = new DownloadRepository();
if (!downloadRepository.Do())
throw new RepositoryRefuseException("Не удалось обновить репозиторий");
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = getRepoGroupsInfo();
break;
case GetAccountQueueSize:
Print("Получить размер очереди для пользователя " + request.arg);
SetCurrentAccountDB(request.arg);
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = (int) account_db.testCompilationTasks.Data.values().stream().filter(ctask -> ctask.state.isActive()).count()
+ (int) account_db.testRunTasks.Data.values().stream().filter(rtask -> rtask.state.isActive()).count();
break;
case GetAccountObjectsCopiesByPKs:
Print("Получить список копий объектов для пользователя " + request.arg);
p = (Pair<Class, Object>) request.object;
SetCurrentAccountDB(request.arg);
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = account_db.getObjectsCopies(p.getKey(), (Vector<Object>) p.getValue());
break;
case GetFirstActiveAccountPackage:
Print("Получить первый активный пакет задач для пользователя " + request.arg);
SetCurrentAccountDB(request.arg);
response = new ServerExchangeUnit_2021(ServerCode.OK);
TasksPackage tasksPackage = account_db.getFirstActivePackage();
LinkedHashMap<Long, TestCompilationTask> activeTasks = account_db.getPackageCompilationTasks(tasksPackage);
response.object = new Pair<>(tasksPackage, activeTasks);
break;
case GetFirstActiveSapforTasksPackage:
Print("Получить первый активный сценарий задач SAPFOR" + request.arg);
SetCurrentAccountDB(request.arg);
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = account_db.getFirstActiveSapforPackage();
break;
case GetQueueSize:
Print("Получить размер глобальной очереди задач");
long date = (long) request.object;
long res = 0;
Vector<String> emails = new Vector<>();
File[] accountsBases_ = Global.DataDirectory.listFiles(pathname ->
pathname.isFile() &&
Utils.getExtension(pathname).equals("sqlite") &&
!Utils.getNameWithoutExtension(pathname.getName()).isEmpty() &&
!pathname.getName().equals(tests_db_name + ".sqlite")
);
if (accountsBases_ != null) {
for (File accountBase : accountsBases_) {
String fileName = accountBase.getName();
String account_email = accountBase.getName().substring(0, fileName.lastIndexOf('_'));
emails.add(account_email);
}
for (String email : emails) {
SetCurrentAccountDB(email);
res += account_db.getQueueSize(date);
}
}
//пройтись по всем аккаунтам, и узнать все пакеты, чья дата меньше равна дате в арге
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = res;
break;
case ReceiveTestsDatabase:
Print("Получить базу данных тестов");
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = Utils.packFile(db.getFile());
break;
case ReceiveTestsTasksDatabase:
Print("Получить базу данных тестовых задач пользователя " + request.arg);
SetCurrentAccountDB(request.arg);
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = Utils.packFile(account_db.getFile());
break;
default:
throw new RepositoryRefuseException("Неподдерживаемый код: " + code);
}
}
}