Merge pull request 'planner_improve' (#1) from planner_improve into main

Reviewed-on: http://alex-freenas.ddns.net:3000/M/VisualSapfor/pulls/1

Planner/Array.h

@@ -1,42 +1,37 @@
+#pragma once 
+
 #include <stdio.h>  
 #include <stdlib.h>  
 #include <string.h> 
-#pragma once 
+#include <vector>
+
 template <class T>
 class Array {
-protected:
+private:
-	long length;  
+	std::vector<T*> elements;
-	T** elements;
+
 public:
-	Array(){
+	Array() { }
-		length=0;
+
-		elements=NULL;
-	}
 	virtual ~Array() {
-		if (elements !=NULL){
+		for (auto& elem : elements)
-			for (long i=0; i<length; ++i)
+			delete elem;
-				delete elements[i];			
+		elements.clear();
-			delete [] elements;
-		}
 	}
+
 	void add(T* new_line) {
-		T ** buf = new T*[length + 1]; 
+		elements.push_back(new_line);
-		for (long i = 0; i < length; ++i) { 
-			buf[i] = elements[i]; 
 	}
-		buf[length] = new_line;  
+
-		length++; 
+	long getLength() const {
-		delete[] elements; 
+		return (long)elements.size();
-		elements = buf; 
-		buf = NULL; 
-	}
-	long getLength(){
-		return length;
 	}
+
 	T* get(long i) {
 		return elements[i];
 	}
-	T** getElements(){
+
+	const std::vector<T*>& getElements() const {
 		return elements;
 	}
 };

Planner/CompilationSupervisor.h

@@ -1,13 +1,15 @@
+#pragma once   
+
 #include "Supervisor.h"   
 #include "CompilationTask.h" 
-#pragma once   
+
 class CompilationSupervisor : public Supervisor<CompilationTask> {
 public:
 	CompilationSupervisor() {
 		this->init("compilationTasks", 4);
 	}
 	CompilationTask* getTaskById(long task_id) {
-		for (long i=0; i< length; ++i){
+		for (long i = 0; i < getLength(); ++i) {
 			CompilationTask* task = get(i);
 			if (task->getId() == task_id)
 				return task;

Planner/CompilationTask.h

@@ -1,5 +1,8 @@
-#include "Task.h"    
 #pragma once    
+
+#include "Task.h"
+#include "Text.h"
+
 class CompilationTask : public Task {
 	String test_id;
 	String makefile_text;
@@ -10,11 +13,12 @@ public:
 	void setMakefileText(String* makefile_text_in) {
 		makefile_text = String(makefile_text_in->getCharArray(), '|');
 	}
-	virtual void print(){	    
+	virtual void print() const {
 		printf("id=%ld; maxtime=%d; test_id=%s\n", id, maxtime,
 			test_id.getCharArray());
 		printf("makefile_text=%s\n", makefile_text.getCharArray());
 	}
+
 	CompilationTask(Text* lines, int offset) :Task(lines, offset) {
 		setTestId(lines->get(offset + 2));
 		setMakefileText(lines->get(offset + 3));

Planner/File.h

@@ -1,5 +1,7 @@
-#include "Text.h" 
 #pragma once
+#include "String.h"
+#include "Text.h"
+
 class File {
 	FILE* ptr;
 public:
@@ -43,7 +45,8 @@ public:
 					line->addChar((char)c);
 					break;
 				}
-			}else { 
+			}
+			else {
 				switch (c) {
 				case '\r':
 					break;

Planner/Global.h

@@ -1,13 +1,10 @@
+#pragma once
+
 #include "String.h"
-#pragma once
+
 String userWorkspace;
-#pragma once
 String packageWorkspace;
-#pragma once
 int maxKernels;
-#pragma once
 int busyKernels;
-#pragma once
 int freeKernels;
-#pragma once
 String dvm_drv;

Planner/Planner.cpp

@@ -1,7 +1,15 @@
+#define _CRT_SECURE_NO_WARNINGS
+using namespace std;
+
+#if __cplusplus >= 201703L
+#include <filesystem>
+#else
+#include <unistd.h>
+#endif
+
 #include "CompilationSupervisor.h"    
 #include "RunSupervisor.h"    
 #include "Global.h"
-#include <unistd.h>
 #include <signal.h>
 
 //https://ru.wikipedia.org/wiki/%D0%A1%D0%B8%D0%B3%D0%BD%D0%B0%D0%BB_(Unix)
@@ -13,6 +21,7 @@ void hdl(int sig)
    fclose(res);
 }
 void set_handlers() {
+#ifndef _WIN32
 	struct sigaction act;
 	memset(&act, 0, sizeof(act));
 	act.sa_handler = hdl;
@@ -73,7 +82,9 @@ void set_handlers(){
 	sigaction(SIGVTALRM, &act, 0);
 	sigaction(SIGXCPU, &act, 0);
 	sigaction(SIGXFSZ, &act, 0);
+#endif
 }
+
 int main(int argc, char ** argv)      
 {
     //+
@@ -87,11 +98,22 @@ int main(int argc, char ** argv)
 	freeKernels = maxKernels;
 	busyKernels= 0;	   
 	//--
+
+#if __cplusplus >= 201703L
+	std::filesystem::current_path(packageWorkspace.getCharArray());
+#else
 	chdir(packageWorkspace.getCharArray());
+#endif
+
 	userWorkspace.println();   
 	packageWorkspace.println();   
 	printf("%d\n", maxKernels);
+#ifndef _WIN32	
 	int pid = getpid();
+#else
+	int pid = _getpid();
+#endif
+
 	printf("PID=%d\n", pid);
 	File pidFile("PID", String(pid));
 	pidFile.Close();
@@ -107,6 +129,6 @@ int main(int argc, char ** argv)
 	RunSupervisor * runSupervisor = new RunSupervisor(compilationSupervisor);   
 	printf("%ld\n", runSupervisor->getLength());	    
 	runSupervisor->print(); 
-	runSupervisor->Do();  
+	runSupervisor->DoWithSchedule(maxKernels);  
 	return 0;      
 }      

Planner/RunSupervisor.h

@@ -1,12 +1,14 @@
+#pragma once
+
 #include "CompilationSupervisor.h"     
 #include "RunTask.h"   
-#pragma once     
+
 class RunSupervisor : public Supervisor<RunTask> {
 public:
 	RunSupervisor(CompilationSupervisor* compilationSupervisor) {
 		this->init("runTasks", 8);
 		//проверить отмененные задачи.  
-		for (long i=0; i< this->length; ++i){  
+		for (long i = 0; i < getLength(); ++i) {
 			RunTask* task = this->get(i);
 			CompilationTask* parent = compilationSupervisor->getTaskById(task->getTestCompilationTaskId());
 			task->setState((parent->getState() == Done) ? Waiting : Canceled);
@@ -20,6 +22,7 @@ public:
 	virtual String getStatePrefix() {
 		return String("Running");
 	}
+
 	/*
 	virtual void Finalize(){
 		this->state = Archivation;

Planner/RunTask.h

@@ -1,5 +1,7 @@
-#include "CompilationTask.h"   
 #pragma once
+
+#include "CompilationTask.h"   
+
 class RunTask : public Task {
 	long testcompilationtask_id;
 	String binary_name;
@@ -9,7 +11,7 @@ class RunTask : public Task {
 	String args;
 	CompilationTask* parent;
 public:
-	virtual void print(){   
+	virtual void print() const {
 		printf("id=%ld; maxtime=%d; testcompilationtask_id=%ld; matrix=%s; environments=%s; usr_par=%s; args=%s kernels=%d\n",
 			id,
 			maxtime,

Planner/String.h

@@ -1,8 +1,10 @@
+#pragma once
+
 #include <stdio.h>     
 #include <stdlib.h>     
 #include <string.h>    
 #include <ctype.h>
-#pragma once     
+
 class String {
 	friend String operator+(const String& a, const String& b);
 	long length;
@@ -15,14 +17,14 @@ public:
 	}
 
 	String(const char* s) {
-		length = strlen(s);     
+		length = (long)strlen(s);
 		body = new char[length + 1];
 		for (long i = 0; i < length; ++i)
 			body[i] = s[i];
 		body[length] = '\0';
 	}
 	String(const char* s, char ps) {
-		length = strlen(s);     
+		length = (long)strlen(s);
 		body = new char[length + 1];
 		for (long i = 0; i < length; ++i) {
 			body[i] = (s[i] == ps) ? '\n' : s[i];
@@ -72,12 +74,15 @@ public:
 			sprintf(body, "%d", s);
 		}
 	}
-	String (long s){   
+
+	String(long s) : String((long long)s) { }
+
+	String(long long s) {
 		length = 0;
 		body = new char[1];
 		body[0] = '\0';
 		if (s >= 0) {
-			long s_ = s;	   
+			long long s_ = s;
 			long size = 1;
 			while (s_ >= 10) {
 				s_ = s_ / 10;
@@ -85,7 +90,7 @@ public:
 			}
 			length = size;
 			body = new char[size + 1];
-			sprintf(body, "%ld", s);   
+			sprintf(body, "%lld", s);
 		}
 	}
 
@@ -144,7 +149,8 @@ public:
 				if (k < s.length) {
 					if (body[i] == s.body[k]) {
 						k++;
-					}else { 
+					}
+					else {
 						//обрыв поиска. 
 						search_on = false;
 						k = 0;
@@ -155,7 +161,8 @@ public:
 					//printf("starts with %ld", start); 
 					return true;
 				}
-			}else { 
+			}
+			else {
 				if (body[i] == s.body[0]) {
 					k = 1;
 					start = i;
@@ -204,7 +211,6 @@ int main(void) {
 		return ( fclose(f) );
 	}
 	 */
-	 
 };
    
 String operator+(const String& a, const String& b) {

Planner/Supervisor.h

@@ -1,8 +1,13 @@
+#pragma once
+
+#include <map>
+#include <set>
+#include <vector>
+#include <queue>
 #include "File.h"   
 #include "Task.h"
-#include <unistd.h>  
+#include "Array.h"
 
-#pragma once     
 enum SupervisorState {   
     WorkspacesCreation, //0   
     Preparation,        //1   
@@ -10,7 +15,7 @@ enum SupervisorState {
     Archivation,        //3
     End         //4   
 };    
-#pragma once     
+  
 template <class T>     
 class Supervisor : public Array <T> {
 protected:
@@ -37,67 +42,26 @@ public:
     }
     //-
     void print() {
-		for (long i=0; i< this->length; ++i)     
+        for (auto& elem : this->getElements())
-			this->elements[i]->print();      
+            elem->print();
     }
+
     void init(const char* fileName, int recordSize) {
         state = WorkspacesCreation;
         File* packedTasks = new File(fileName);
         Text* lines = packedTasks->readLines();
-		this->length = lines->getLength()/recordSize;    
+
-		this->elements = new T* [this->length];       
+        const long length = lines->getLength() / recordSize;
         int offset = 0;
-		for (int i=0; i< this->length; ++i){   
+        for (int i = 0; i < length; ++i) {
-			this->elements[i]= new T(lines, offset);  
+            this->add(new T(lines, offset));
             offset += recordSize;
         }
         delete packedTasks;
         delete lines;
     }
-	void Do(){   
+
-		saveState();
+    void changeState() {
-		long activeCount=0;	   
-		//todo обязательно убрать отладочную печать.  
-		printf("tasks count = %ld\n", this->length);	   
-		while (this->state!= End){   
-//			printf("state=%d\n", this->state);  
-//			printf("max=%d; busy=%d; free=%d\n", maxKernels, busyKernels, freeKernels);			   
-			activeCount=0;   
-			for (long i=0; i<this->length; ++i){   
-				T * task = this->elements[i]; 				  
-				switch (this->state){   
-					case WorkspacesCreation:   
-						if (task->getState()==Waiting){   
-							activeCount++;   
-							task->createWorkspace(); 		
-							task->setState(WorkspaceCreated);					  
-						}   
-						break; 						  
-					case Preparation:   
-						if (task->getState()==WorkspaceCreated){   
-							activeCount++;   
-							task->prepareWorkspace();   
-							task->createLaunchScript();
-							task->setState(WorkspaceReady);							  
-						}   
-						break;						  
-					case Execution:  
-						if (task->getState()==WorkspaceReady){  
-							activeCount++;
-							task->start_time=Utils::getAbsoluteTime();
-							task->Start();							  
-						}else if (task->getState()==Running){  
-							activeCount++;  
-							task->Check();  
-						}														  
-						break;  
-					default:   
-//						printf("id = %ld; state = %d\n", task->getId(), task->getState());   
-						break;   
-				}   
-			}   
-//			printf("active count = %d\n", activeCount);   
-			if (activeCount==0){				   
         switch (this->state) {
         case WorkspacesCreation:
             this->state = Preparation;
@@ -117,10 +81,162 @@ public:
             break;
         }
     }
+
+    void Do() {
+        saveState();
+        long activeCount = 0;
+        //todo обязательно убрать отладочную печать.  
+        printf("tasks count = %ld\n", this->getLength());
+        while (this->state != End) {
+            //			printf("state=%d\n", this->state);  
+            //			printf("max=%d; busy=%d; free=%d\n", maxKernels, busyKernels, freeKernels);			   
+            activeCount = 0;
+            for (long i = 0; i < this->getLength(); ++i) {
+                T* task = this->get(i);
+                switch (this->state) {
+                case WorkspacesCreation:
+                    if (task->getState() == Waiting) {
+                        activeCount++;
+                        task->createWorkspace();
+                        task->setState(WorkspaceCreated);
+                    }
+                    break;
+                case Preparation:
+                    if (task->getState() == WorkspaceCreated) {
+                        activeCount++;
+                        task->prepareWorkspace();
+                        task->createLaunchScript();
+                        task->setState(WorkspaceReady);
+                    }
+                    break;
+                case Execution:
+                    if (task->getState() == WorkspaceReady) {
+                        activeCount++;
+                        task->Start();
+                    }
+                    else if (task->getState() == Running) {
+                        activeCount++;
+                        task->Check();
+                    }
+                    break;
+                default:
+                    //						printf("id = %ld; state = %d\n", task->getId(), task->getState());   
+                    break;
+                }
+            }
+            //			printf("active count = %d\n", activeCount);   
+            if (activeCount == 0)
+                changeState();
             Utils::Sleep(2);
         }
     }
+
+
+    void DoWithSchedule(int maxKernels) {
+        saveState();
+
+        // подготовка тестов
+        while (this->state != Execution) {
+            for (auto& task : this->getElements()) {
+                switch (this->state) {
+                case WorkspacesCreation:
+                    if (task->getState() == Waiting) {
+                        task->createWorkspace();
+                        task->setState(WorkspaceCreated);
+                    }
+                    break;
+                case Preparation:
+                    if (task->getState() == WorkspaceCreated) {
+                        task->prepareWorkspace();
+                        task->createLaunchScript();
+                        task->setState(WorkspaceReady);
+                    }
+                    break;
+                default:
+                    //printf("id = %ld; state = %d\n", task->getId(), task->getState());   
+                    break;
+                }
+            }
+            changeState();
+        }
+
+        map<int, queue<T*>, std::greater<int>> sortedByKernelNeeds;
+
+        long activeTasks = 0;
+        long done = 0;
+        for (auto& task : this->getElements()) {
+            if (task->getState() == WorkspaceReady) {
+                activeTasks++;
+                sortedByKernelNeeds[task->getKernels()].push(task);
+            }
+        }
+
+        printf("total tasks count = %ld, active task count %ld, maxkernels %d\n", this->getLength(), activeTasks, maxKernels);
+
+        int busyKernels = 0;
+        set<T*> activeTaskSet;
+        bool ignoreCheck = true;
+
+        while (activeTasks) {
+            long oldActiveTasks = activeTasks;
+            vector<int> emptyKeys;
+
+            //ставим задачи от больших к меньшему по ядрам
+            for (auto& elem : sortedByKernelNeeds) {
+                int freeKernels = maxKernels - busyKernels;
+                int kernelsNeeded = elem.first;
+
+                while (kernelsNeeded <= freeKernels && elem.second.size()) {
+                    T* task = elem.second.front();
+                    elem.second.pop();
+
+                    activeTaskSet.insert(task);
+                    task->Start(ignoreCheck);
+                    printf("start task with %d kernels and id %ld\n", task->getKernels(), task->getId());
+
+                    busyKernels += task->getKernels();
+                    freeKernels = maxKernels - busyKernels;
+                }
+
+                if (elem.second.size() == 0)
+                    emptyKeys.push_back(kernelsNeeded);
+
+                //если ядер не осталось, то нет смысла дальше смотреть
+                if (freeKernels == 0)
+                    break;
+            }
+
+            // очищаем от пустых ключей
+            for (auto& empty : emptyKeys)
+                sortedByKernelNeeds.erase(empty);
+
+            // проверяем нет ли завершившихся задач
+            for (auto it = activeTaskSet.begin(); it != activeTaskSet.end(); )
+            {
+                T* task = *(it);
+
+                if (task->Check()) {
+                    it++;
+                    activeTaskSet.erase(task);
+                    activeTasks--;
+                    done++;
+                    busyKernels -= task->getKernels();
+                    printf("  done task with %d kernels and id %ld\n", task->getKernels(), task->getId());
+
+                    continue;
+                }
+                it++;
+            }
+
+            if (oldActiveTasks != activeTasks)
+                printf("done %ld / %ld\n", done, this->getLength());
+        }
+
+        changeState();
+    }
+
     virtual void Finalize() {}
+
     void saveState() {
         Utils::Sleep(1); //чтобы не было одинаковых по дате файлов.
         String stateFile = packageWorkspace + "/state/" + getStatePrefix() + printState();

Planner/Task.h

@@ -1,8 +1,11 @@
+#pragma once    
+
+#include <chrono>
+
 #include "File.h"  
 #include "Utils.h"  
 #include "Global.h" 
 
-#pragma once    
 enum TaskState {
 	Inactive, //0 
 	Waiting,  //1 
@@ -25,14 +28,13 @@ enum TaskState {
 	FailedToQueue,  //18 
 	AbortingByUser  //19 
 };
-#pragma once
+
 enum TestType {
 	Default, //0
 	Correctness, //1
 	Performance, //2
 };
 
-#pragma once    
 class Task {
 protected:
 	long id;
@@ -41,8 +43,9 @@ protected:
 	int kernels; //получение зависит от типа задачи.
 	String workspace;
 	TaskState state;
+	std::chrono::system_clock::time_point start_time;
+
 public:
-   long start_time;
 	String printState() {
 		switch (state) {
 		case Inactive:
@@ -89,7 +92,10 @@ public:
 			return "?";
 		}
 	}
+
 	//-------------->> 
+	void setStart() { start_time = std::chrono::system_clock::now(); }
+
 	long getId() { return id; }
 	long setId(String* id_s) {
 		return id = strtol(id_s->getCharArray(), NULL, 10);
@@ -106,7 +112,7 @@ public:
 		setMaxtime(lines->get(offset + 1));
 		workspace = packageWorkspace + "/" + String(id);
 	}
-	virtual void print()=0;   
+	virtual void print() const = 0;
 	//- 	 
 	virtual void prepareWorkspace() {}
 	virtual String getLaunchScriptText() = 0;
@@ -125,9 +131,12 @@ public:
 		File launchScriptFile = File(launchScriptPath, launchScriptText);
 		Utils::Chmod(launchScriptPath);
 	}
-	virtual void Start(){		 
 
-		if (kernels<=freeKernels){		 
+	int getKernels() const { return kernels; }
+
+	void Start(bool dontCheck = false) {		
+		if (kernels <= freeKernels || dontCheck) {
+			setStart();
 			system(getStartCommand().getCharArray());
 			state = Running;
 			//- 
@@ -136,36 +145,47 @@ public:
 			//- 
 		}
 	}
-	virtual void analyseResults(){
+
-		state=Finished;
+	//return 'true' if done, 'false' - if running
-	}
+	virtual bool Check() {
-	virtual void Check(){ 
+		if (Utils::Exists(workspace + "/DONE"))
-		if (Utils::Exists(workspace+"/DONE")){ 
 			analyseResults();
-		}else { 
+		else {
 			if (Utils::Exists(workspace + "/TIMEOUT")) {
 				state = AbortedByTimeout;
 				//todo определить по интервалу времени на всякий случай.
-			}else if (Utils::Exists(workspace+"/INTERRUPT")){ 
+			}
+			else if (Utils::Exists(workspace + "/INTERRUPT")) {
 				state = AbortedByUser;
-			} else {
+			}
-			    long now = Utils::getAbsoluteTime();
+			else {
-			    long delta = now-start_time;
+				auto end_time = std::chrono::system_clock::now();
-			    if (maxtime<delta){
+				std::chrono::duration<double> diff_time = end_time - start_time;
+
+				if (maxtime * 1.1 < diff_time.count()) {
+					printf("SET STATUS ABORTED by timer to %ld with time %f sec / %f sec", id, diff_time.count(), maxtime * 1.1);
 					state = AbortedByTimeout;
 				}
 			}
 		}
+
 		if (state != Running) {
 			//- 
 			busyKernels = Utils::min(busyKernels - kernels, maxKernels);
 			freeKernels = Utils::max(0, maxKernels - busyKernels);
 			//- 
-			saveState(); //не нужно. только для отладки. анализ будет делаться архивом.
+			saveState();
 		}
+
+		return (state != Running);
 	}
+
+	virtual void analyseResults() {
+		state = Finished;
+	}
+
 	virtual void saveState() {
 		String stateFile = workspace + "/TaskState";
-		File(stateFile, printState()); 
+		File tmp(stateFile, printState());
 	}
 };

Planner/Text.h

@@ -1,20 +1,24 @@
+#pragma once 
+
 #include "String.h" 
 #include "Array.h"
-#pragma once 
+
 class Text : public Array<String> {
 public:
-	void Print(){
+	void Print() const {
-		printf("text length=%ld\n", length);
+		printf("text length=%ld\n", this->getLength());
 
-		for (long i=0; i<length; ++i){
+		auto elems = this->getElements();
-			printf("i=%ld; [%s]\n",i, elements[i]->getCharArray());
+		for (long i = 0; i < elems.size(); ++i) {
+			printf("i=%ld; [%s]\n", i, elems[i]->getCharArray());
 			//		elements[i]->println();
 		}
 	}
-	bool hasMatch(const String& s){
 
-		for (long i=0; i<length; ++i){
+	bool hasMatch(const String& s) const {
-			if (s.contains(*elements[i])){
+
+		for (auto& elem : this->getElements()) {
+			if (s.contains(*elem)) {
 				//printf("match: [%s]\n", elements[i]->getCharArray());
 				return true;
 			}

Planner/Utils.h

@@ -1,10 +1,18 @@
+#pragma once
+
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <errno.h>
-#include <unistd.h>
 #include <time.h>
+#include <chrono>
+#include <thread>
+
+#if __cplusplus >= 201703L
+#include <filesystem>
+#endif
+
 #include "String.h"
-#pragma once
+
 class Utils {
 public:
 	static int max(int a, int b) {
@@ -14,31 +22,39 @@ public:
 		return (a > b) ? b : a;
 	}
 	static void Mkdir(const String& path) {
+#if __cplusplus >= 201703L
+		std::filesystem::create_directory(path.getCharArray());
+#else
 		mkdir(path.getCharArray(), 0777);
+#endif
 	}
+
 	//https://stackoverflow.com/questions/4568681/using-chmod-in-a-c-program
 	static void Chmod(const String& path) {
 		String command = "chmod 777 " + String::DQuotes(path);
 		system(command.getCharArray());
 	}
+
 	//https://stackoverflow.com/questions/230062/whats-the-best-way-to-check-if-a-file-exists-in-c
 	static bool Exists(const String& path) {
 		struct stat buffer;
 		return (stat(path.getCharArray(), &buffer) == 0);
 	}
+
+	//in seconds
 	static void Sleep(int s) {
-		usleep(s* 1000000);
+		std::chrono::seconds timespan(s);
+		std::this_thread::sleep_for(timespan);
 	}
 	static void Copy(const String& src, const String& dst) {
 		String command = "cp " + String::DQuotes(src) + " " + String::DQuotes(dst);
 		system(command.getCharArray());
 	}
-	static long getAbsoluteTime(){
+	static time_t getAbsoluteTime() {
 		return time(NULL);
 	}
 	static String getDate() {
-		long int ttime;
+		auto ttime = time(NULL);
-   		ttime = time (NULL);
 		String res(ctime(&ttime));
 		return res;
 	}