Перенос.

This commit is contained in:
2023-09-17 22:13:42 +03:00
parent dd2e0ca7e0
commit 629d8b8477
1239 changed files with 61161 additions and 1 deletions

42
Planner/Array.h Normal file
View File

@@ -0,0 +1,42 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#pragma once
template <class T>
class Array {
protected:
long length;
T** elements;
public:
Array(){
length=0;
elements=NULL;
}
virtual ~Array(){
if (elements !=NULL){
for (long i=0; i<length; ++i)
delete elements[i];
delete [] elements;
}
}
void add(T * new_line) {
T ** buf = new T*[length + 1];
for (long i = 0; i < length; ++i) {
buf[i] = elements[i];
}
buf[length] = new_line;
length++;
delete[] elements;
elements = buf;
buf = NULL;
}
long getLength(){
return length;
}
T * get(long i){
return elements[i];
}
T** getElements(){
return elements;
}
};

View File

@@ -0,0 +1,20 @@
#include "Supervisor.h"
#include "CompilationTask.h"
#pragma once
class CompilationSupervisor: public Supervisor<CompilationTask> {
public:
CompilationSupervisor(){
this->init("compilationTasks", 4);
}
CompilationTask * getTaskById(long task_id){
for (long i=0; i< length; ++i){
CompilationTask * task = get(i);
if (task->getId()==task_id)
return task;
}
return NULL;
}
virtual String getStatePrefix(){
return String("Compilation");
}
};

47
Planner/CompilationTask.h Normal file
View File

@@ -0,0 +1,47 @@
#include "Task.h"
#pragma once
class CompilationTask: public Task {
String test_id;
String makefile_text;
public:
void setTestId(String * test_id_in){
test_id = String(test_id_in->getCharArray());
}
void setMakefileText(String * makefile_text_in){
makefile_text = String(makefile_text_in->getCharArray(), '|');
}
virtual void print(){
printf("id=%ld; maxtime=%d; test_id=%s\n", id, maxtime,
test_id.getCharArray());
printf("makefile_text=%s\n", makefile_text.getCharArray());
}
CompilationTask(Text * lines, int offset):Task(lines,offset) {
setTestId(lines->get(offset+2));
setMakefileText(lines->get(offset+3));
setState(Waiting);
kernels=1;
}
virtual void prepareWorkspace(){
String makeFilePath = String(id)+"/Makefile";
File makeFileFile = File(makeFilePath, this->makefile_text);
String tests = userWorkspace+"/projects";
String testPath= tests+"/"+test_id;
String copyCommand = "cp -r " + String::DQuotes(testPath + "/.") + " "+ String::DQuotes(workspace);
system(copyCommand.getCharArray());
}
virtual String getLaunchScriptText(){
String modules = userWorkspace+"/modules";
String starterCall = modules+"/starter";
String launcherCall = modules+"/launcher";
return String::DQuotes(starterCall)+" "+
String::DQuotes(launcherCall)+" "+
String(maxtime)+" "+
String::DQuotes("")+" "+
"make -j -f Makefile";
}
virtual void analyseResults(){
String binary = workspace+"/0";
state = Utils::Exists(binary)? Done:DoneWithErrors;
}
};

64
Planner/File.h Normal file
View File

@@ -0,0 +1,64 @@
#include "Text.h"
#pragma once
class File {
FILE* ptr;
public:
File(String* name) {
ptr = fopen(name->getCharArray(), "r");
}
File(const char * name) {
ptr = fopen(name, "r");
}
File(const String& name, const String& text){
ptr = fopen(name.getCharArray(), "w");
fprintf(ptr, "%s\n", text.getCharArray());
}
~File() {
if (ptr != NULL) {
fclose(ptr);
ptr = NULL;
}
}
Text* readLines(){
Text* lines = new Text();
int c;
String * line = NULL;
bool lineStarted = false;
do {
c = fgetc(ptr);
if (lineStarted){
switch (c) {
case '\r':
break;
case '\n':
case EOF:
lines->add(line);
line = NULL;
lineStarted = false;
break;
default:
line->addChar((char)c);
break;
}
}else {
switch (c){
case '\r':
break;
case '\n':
line = new String();
lines->add(line);
line = NULL;
break;
case EOF:
break;
default:
line = new String();
line->addChar((char)c);
lineStarted = true;
break;
}
}
}while (c!=EOF);
return lines;
}
};

13
Planner/Global.h Normal file
View File

@@ -0,0 +1,13 @@
#include "String.h"
#pragma once
String userWorkspace;
#pragma once
String packageWorkspace;
#pragma once
int maxKernels;
#pragma once
int busyKernels;
#pragma once
int freeKernels;
#pragma once
String dvm_drv;

28
Planner/Planner.cpp Normal file
View File

@@ -0,0 +1,28 @@
#include "CompilationSupervisor.h"
#include "RunSupervisor.h"
#include "Global.h"
int main(int argc, char ** argv)
{
userWorkspace = String(argv[1]);
packageWorkspace = String(argv[2]);
maxKernels = atoi(argv[3]);
dvm_drv = String(argv[4]);
//--
freeKernels = maxKernels;
busyKernels= 0;
//--
chdir(packageWorkspace.getCharArray());
userWorkspace.println();
packageWorkspace.println();
printf("%d\n", maxKernels);
CompilationSupervisor * compilationSupervisor = new CompilationSupervisor();
printf("%ld\n", compilationSupervisor->getLength());
compilationSupervisor->Do();
RunSupervisor * runSupervisor = new RunSupervisor(compilationSupervisor);
printf("%ld\n", runSupervisor->getLength());
runSupervisor->print();
runSupervisor->Do();
return 0;
}

31
Planner/RunSupervisor.h Normal file
View File

@@ -0,0 +1,31 @@
#include "CompilationSupervisor.h"
#include "RunTask.h"
#pragma once
class RunSupervisor: public Supervisor<RunTask> {
public:
RunSupervisor(CompilationSupervisor * compilationSupervisor){
this->init("runTasks", 8);
//проверить отмененные задачи.
for (long i=0; i< this->length; ++i){
RunTask * task = this->get(i);
CompilationTask * parent = compilationSupervisor->getTaskById( task->getTestCompilationTaskId());
task->setState((parent->getState()==Done)?Waiting:Canceled);
task->setParent(parent);
printf("id=%ld; parent_id = %ld; state=%s\n",
task->getId(),
task->getParent()->getId(),
task->printState().getCharArray());
}
}
virtual String getStatePrefix(){
return String("Running");
}
virtual void Finalize(){
this->state = Archivation;
saveState();
printf("Archivation started\n");
Utils::ZipFolder(String("./"),String("archive.zip"));
printf("Archivation ended\n");
}
};

94
Planner/RunTask.h Normal file
View File

@@ -0,0 +1,94 @@
#include "CompilationTask.h"
#pragma once
class RunTask : public Task {
long testcompilationtask_id;
String binary_name;
String matrix;
String environments;
String usr_par;
String args;
CompilationTask* parent;
public:
virtual void print(){
printf("id=%ld; maxtime=%d; testcompilationtask_id=%ld; matrix=%s; environments=%s; usr_par=%s; args=%s kernels=%d\n",
id,
maxtime,
testcompilationtask_id,
matrix.getCharArray(),
environments.getCharArray(),
usr_par.getCharArray(),
args.getCharArray(),
kernels
);
}
int setKernels(String * kernels_s){
return kernels = atoi(kernels_s->getCharArray());
}
long setTestCompilationTaskId(String * id_s){
return testcompilationtask_id=strtol(id_s->getCharArray(), NULL, 10);
}
long getTestCompilationTaskId(){
return testcompilationtask_id;
}
void setMatrix(String * matrix_in){
matrix= String (matrix_in->getCharArray());
}
void setEnvironments(String * environments_in){
environments= String(environments_in->getCharArray());
}
void setUsrPar(String * usr_par_in){
usr_par= String(usr_par_in->getCharArray(),'|');
}
void setArgs(String * args_in){
args= String(args_in->getCharArray());
}
void setParent(CompilationTask * parent_in){
parent = parent_in;
binary_name = "spf_"+ String(id)+"_"+matrix.Replace(' ','_');
}
CompilationTask * getParent(){
return parent;
}
RunTask(Text * lines, int offset):Task(lines,offset) {
setTestCompilationTaskId(lines->get(offset+2));
setMatrix(lines->get(offset+3));
setEnvironments(lines->get(offset+4));
setUsrPar(lines->get(offset+5));
setArgs(lines->get(offset+6));
setKernels(lines->get(offset+7));
}
virtual String getLaunchScriptText(){
String modules = userWorkspace+"/modules";
String starterCall = modules+"/starter";
String launcherCall = modules+"/launcher";
//-
String dvm_start = String::DQuotes(dvm_drv) + " run ";
if (!matrix.isEmpty())
dvm_start = dvm_start+matrix + " ";
dvm_start = dvm_start+ String::DQuotes("./" + binary_name);
if (!args.isEmpty())
dvm_start = dvm_start+ " " + args;
return String::DQuotes(starterCall)+" "+
String::DQuotes(launcherCall)+" "+
String(maxtime)+" "+
String::DQuotes("killall -SIGKILL " + binary_name)+" "+
dvm_start;
}
virtual void prepareWorkspace(){
String binary_src = parent->getWorkspace()+"/0";
String binary_dst = workspace+"/"+binary_name;
Utils::Copy(binary_src, binary_dst);
if (!usr_par.isEmpty()){
String parPath = String(id)+"/usr.par";
File parFile = File(parPath, usr_par);
}
}
virtual String getStartCommand(){
String res = workspace+"/run";
if (!environments.isEmpty())
res = environments+" "+res;
printf("START %ld: %s\n", id, res.getCharArray());
return res;
}
};

220
Planner/String.h Normal file
View File

@@ -0,0 +1,220 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#pragma once
class String {
friend String operator+(const String& a, const String& b);
long length;
char* body;
public:
String() {
length = 0;
body = new char[1];
body[0] = '\0';
}
String(const char* s) {
length = strlen(s);
body = new char[length + 1];
for (long i = 0; i < length; ++i)
body[i] = s[i];
body[length]='\0';
}
String(const char* s, char ps) {
length = strlen(s);
body = new char[length + 1];
for (long i = 0; i < length; ++i){
body[i] = (s[i]==ps)? '\n': s[i];
}
body[length]='\0';
}
~String() {
if (body != NULL) {
delete[] body;
}
}
void println() const{
printf("[%s]\n", body);
}
void addChar(char c) {
char* buf = new char[length + 2];
for (long i = 0; i < length; ++i) {
buf[i] = body[i];
}
buf[length] = c;
length++;
//--
buf[length] = '\0';
delete[] body;
body = buf;
buf = NULL;
}
char * getCharArray() const{
return body;
}
String (int s){
length = 0;
body = new char[1];
body[0] = '\0';
if (s>=0){
int s_ = s;
int size = 1;
while (s_>=10){
s_ = s_/10;
size++;
}
length = size;
body = new char [size+1];
sprintf(body, "%d", s);
}
}
String (long s){
length = 0;
body = new char[1];
body[0] = '\0';
if (s>=0){
long s_ = s;
long size = 1;
while (s_>=10){
s_ = s_/10;
size++;
}
length = size;
body = new char [size+1];
sprintf(body, "%ld", s);
}
}
const String& operator=(const String& s){
if (body != NULL)
delete[] body;
length = s.length;
body = new char[length+1];
for (long i=0; i<length; ++i)
body[i]=s.body[i];
body[length]='\0';
return *this;
}
static String DQuotes(const String& s){
String res;
res.length = s.length+2;
res.body=new char[res.length+1];
res.body[0] = '"';
res.body[res.length-1] = '"';
res.body[res.length] = '\0';
for (long i=0; i< s.length; ++i)
res.body[i+1]=s.body[i];
return res;
}
String Replace(char f, char t){
String res;
res.length = length;
res.body = new char[length];
res.body[length]='\0';
for (long i = 0; i < length; ++i)
res.body[i]= (body[i]==f)? t: body[i];
return res;
}
String Remove(char f){
String res;
for (long i=0; i<length; ++i){
if (body[i]!=f)
res.addChar(body[i]);
}
return res;
}
bool isEmpty() const{
return length==0;
}
bool contains(const String& s) const{
bool res = false;
bool search_on=false;
if (s.isEmpty()) return true;
if (s.length>length) return false;
long k=0;
long start=-1;
for (long i=0; i<length; ++i){
if (search_on){
if (k<s.length){
if (body[i]==s.body[k]){
k++;
}else {
//обрыв поиска.
search_on =false;
k=0;
start=-1;
}
}
else {
//printf("starts with %ld", start);
return true;
}
}else {
if (body[i]== s.body[0]){
k=1;
start=i;
search_on=true;
//printf("search started %d\n", start);
}
}
}
if (search_on) {
//printf("starts with %ld\n", start);
res=true;
}
return res;
}
String toUpper(){
String res = String(this->getCharArray());
for (long i=0; i<length; ++i)
res.body[i]= toupper(body[i]);
res.println();
return res;
}
long getLength(){
return length;
}
bool operator==(const String&s) const {
if (length!=s.length) return false;
for (long i=0; i<length; ++i){
if (body[i]!=s.body[i]) return false;
}
return true;
}
/* регистр.
#include <stdio.h>
#include <ctype.h>
int main(void) {
FILE * f;
int c;
if ( ! ( f = fopen("file.txt", "r") ) )
return -1;
while ( ( c = fgetc(f) ) != EOF )
putchar( isupper(c) ? tolower(c) : toupper(c) );
return ( fclose(f) );
}
*/
};
String operator+(const String& a, const String& b){
String res = String();
res.length =a.length+b.length;
res.body = new char[res.length+1];
for (long i=0; i<a.length; ++i)
res.body[i]=a.body[i];
for (long i=0; i<b.length; ++i)
res.body[i+a.length]=b.body[i];
res.body[res.length]='\0';
return res;
}

129
Planner/Supervisor.h Normal file
View File

@@ -0,0 +1,129 @@
#include "File.h"
#include "Task.h"
#include <unistd.h>
#pragma once
enum SupervisorState {
WorkspacesCreation, //0
Preparation, //1
Execution, //2
Archivation, //3
End //4
};
#pragma once
template <class T>
class Supervisor : public Array <T> {
protected:
SupervisorState state;
public:
virtual String getStatePrefix(){
return String("");
}
String printState(){
switch(state){
case WorkspacesCreation:
return String("WorkspacesCreation");
case Preparation:
return String("Preparation");
case Execution:
return String("Execution");
case Archivation:
return String("Archivation");
case End:
return String("End");
default:
return "?";
}
}
//-
void print(){
for (long i=0; i< this->length; ++i)
this->elements[i]->print();
}
void init(const char * fileName, int recordSize){
state = WorkspacesCreation;
File * packedTasks = new File(fileName);
Text * lines = packedTasks->readLines();
this->length = lines->getLength()/recordSize;
this->elements = new T* [this->length];
int offset=0;
for (int i=0; i< this->length; ++i){
this->elements[i]= new T(lines, offset);
offset+=recordSize;
}
delete packedTasks;
delete lines;
}
void Do(){
saveState();
long activeCount=0;
//todo обязательно убрать отладочную печать.
printf("tasks count = %ld\n", this->length);
while (this->state!= End){
// printf("state=%d\n", this->state);
// printf("max=%d; busy=%d; free=%d\n", maxKernels, busyKernels, freeKernels);
activeCount=0;
for (long i=0; i<this->length; ++i){
T * task = this->elements[i];
switch (this->state){
case WorkspacesCreation:
if (task->getState()==Waiting){
activeCount++;
task->createWorkspace();
task->setState(WorkspaceCreated);
}
break;
case Preparation:
if (task->getState()==WorkspaceCreated){
activeCount++;
task->prepareWorkspace();
task->createLaunchScript();
task->setState(WorkspaceReady);
}
break;
case Execution:
if (task->getState()==WorkspaceReady){
activeCount++;
task->Start();
}else if (task->getState()==Running){
activeCount++;
task->Check();
}
break;
default:
// printf("id = %ld; state = %d\n", task->getId(), task->getState());
break;
}
}
// printf("active count = %d\n", activeCount);
if (activeCount==0){
switch (this->state){
case WorkspacesCreation:
this->state = Preparation;
saveState();
break;
case Preparation:
this->state = Execution;
saveState();
break;
case Execution:
Finalize();
this->state = End;
saveState();
break;
default:
this->state = End;
break;
}
}
Utils::Sleep(2);
}
}
virtual void Finalize(){}
void saveState(){
Utils::Sleep(1); //чтобы не было одинаковых по дате файлов.
String stateFile = packageWorkspace+"/state/"+getStatePrefix()+printState();
//printf("stateFile=<%s>\n", stateFile.getCharArray());
File(stateFile, Utils::getDate());
}
};

163
Planner/Task.h Normal file
View File

@@ -0,0 +1,163 @@
#include "File.h"
#include "Utils.h"
#include "Global.h"
#pragma once
enum TaskState {
Inactive, //0
Waiting, //1
WorkspaceCreated, //2
WorkspaceReady, //3
Running, //4
Canceled, //5
Finished, //6
FinishedAbortedByTimeout, //7
FinishedAbortedByUser, //8
Done, //9
DoneWithErrors, //10
AbortedByTimeout, //11
AbortedByUser, //12
Crushed, //13
WrongTestFormat, //14
InternalError, //15
Queued, //16
NoSuchTask, //17
FailedToQueue, //18
AbortingByUser //19
};
#pragma once
enum TestType{
Default, //0
Correctness, //1
Performance, //2
};
#pragma once
class Task {
protected:
long id;
int maxtime;
int kernels; //получение зависит от типа задачи.
String workspace;
TaskState state;
public:
String printState(){
switch(state){
case Inactive:
return String("Inactive");
case Waiting:
return String("Waiting");
case WorkspaceCreated:
return String("WorkspaceCreated");
case WorkspaceReady:
return String("WorkspaceReady");
case Running:
return String("Running");
case Canceled:
return String("Canceled");
case Finished:
return String("Finished");
case FinishedAbortedByTimeout:
return String("FinishedAbortedByTimeout");
case FinishedAbortedByUser:
return String("FinishedAbortedByUser");
case Done:
return String("Done");
case DoneWithErrors:
return String("DoneWithErrors");
case AbortedByTimeout:
return String("AbortedByTimeout");
case AbortedByUser:
return String("AbortedByUser");
case Crushed:
return String("Crushed");
case WrongTestFormat:
return String("WrongTestFormat");
case InternalError:
return String("InternalError");
case Queued:
return String("Queued");
case NoSuchTask:
return String("NoSuchTask");
case FailedToQueue:
return String("FailedToQueue");
case AbortingByUser:
return String("AbortingByUser");
default:
return "?";
}
}
//-------------->>
long getId(){return id;}
long setId(String * id_s){
return id=strtol(id_s->getCharArray(), NULL, 10);
}
int getMaxtime(){return maxtime;}
int setMaxtime(String * maxtime_s){
return maxtime=atoi(maxtime_s->getCharArray());
}
const String& getWorkspace(){return workspace;}
TaskState getState(){return state;}
TaskState setState(TaskState state_in){return state=state_in;}
Task(Text * lines, int offset){
setId(lines->get(offset));
setMaxtime(lines->get(offset+1));
workspace = packageWorkspace+"/"+String(id);
}
virtual void print()=0;
//-
virtual void prepareWorkspace(){}
virtual String getLaunchScriptText()=0;
virtual String getStartCommand(){
return workspace+"/run";
}
void createWorkspace(){
Utils::Mkdir(workspace);
}
void createLaunchScript(){
String launchScriptPath = workspace+"/run";
String launchScriptText =
String("cd ")+String::DQuotes(workspace)+"\n"+
getLaunchScriptText();
File launchScriptFile = File(launchScriptPath, launchScriptText);
Utils::Chmod(launchScriptPath);
}
virtual void Start(){
if (kernels<=freeKernels){
system(getStartCommand().getCharArray());
state=Running;
//-
busyKernels= Utils::min(busyKernels+kernels, maxKernels);
freeKernels= Utils::max(0, maxKernels-busyKernels);
//-
}
}
virtual void analyseResults(){
state=Finished;
}
virtual void Check(){
if (Utils::Exists(workspace+"/DONE")){
analyseResults();
}else {
if (Utils::Exists(workspace+"/TIMEOUT")){
state=AbortedByTimeout;
//todo определить по интервалу времени на всякий случай.
}else if (Utils::Exists(workspace+"/INTERRUPT")){
state=AbortedByUser;
}
}
if (state!=Running){
//-
busyKernels= Utils::min(busyKernels-kernels, maxKernels);
freeKernels= Utils::max(0, maxKernels-busyKernels);
//-
saveState(); //не нужно. только для отладки. анализ будет делаться архивом.
}
}
virtual void saveState(){
String stateFile = workspace+"/TaskState";
File(stateFile, printState());
}
};

25
Planner/Text.h Normal file
View File

@@ -0,0 +1,25 @@
#include "String.h"
#include "Array.h"
#pragma once
class Text: public Array<String> {
public:
void Print(){
printf("text length=%ld\n", length);
for (long i=0; i<length; ++i){
printf("i=%ld; [%s]\n",i, elements[i]->getCharArray());
// elements[i]->println();
}
}
bool hasMatch(const String& s){
for (long i=0; i<length; ++i){
if (s.contains(*elements[i])){
//printf("match: [%s]\n", elements[i]->getCharArray());
return true;
}
}
//printf("no matches for [%s]\n", s.getCharArray());
return false;
}
};

49
Planner/Utils.h Normal file
View File

@@ -0,0 +1,49 @@
#include <sys/stat.h>
#include <sys/types.h>
#include <errno.h>
#include <unistd.h>
#include <time.h>
#include "String.h"
#pragma once
class Utils {
public:
static int max(int a, int b){
return (a > b)? a:b;
}
static int min(int a, int b){
return (a > b)? b:a;
}
static void Mkdir(const String& path){
mkdir(path.getCharArray(), 0777);
}
//https://stackoverflow.com/questions/4568681/using-chmod-in-a-c-program
static void Chmod(const String& path){
String command = "chmod 777 "+String::DQuotes(path);
system(command.getCharArray());
}
//https://stackoverflow.com/questions/230062/whats-the-best-way-to-check-if-a-file-exists-in-c
static bool Exists(const String& path){
struct stat buffer;
return (stat (path.getCharArray(), &buffer) == 0);
}
static void Sleep(int s){
usleep(s* 1000000);
}
static void Copy(const String& src, const String& dst){
String command = "cp "+String::DQuotes(src)+" "+String::DQuotes(dst);
system(command.getCharArray());
}
static long getAbsoluteTime(){
return time (NULL);
}
static String getDate(){
long int ttime;
ttime = time (NULL);
String res(ctime (&ttime));
return res;
}
static void ZipFolder(const String& src, const String& dst){
String command = "zip -r "+String::DQuotes(dst)+" "+String::DQuotes(src);
system(command.getCharArray());
}
};