Merge branch 'feature/percentage-helper' into 'dev'

Add percentage to backups/support logs page

See merge request crafty-controller/crafty-commander!205

app/classes/controllers/users_controller.py

@@ -107,6 +107,14 @@ class Users_Controller:
     def user_id_exists(user_id):
         return users_helper.user_id_exists(user_id)
 
+    @staticmethod
+    def set_prepare(user_id):
+        return users_helper.set_prepare(user_id)
+
+    @staticmethod
+    def stop_prepare(user_id):
+        return users_helper.stop_prepare(user_id)
+
     @staticmethod
     def get_user_id_by_api_token(token: str) -> str:
         token_data = authentication.check_no_iat(token)

app/classes/models/users.py

@@ -37,6 +37,7 @@ class Users(Model):
     support_logs = CharField(default = '')
     valid_tokens_from = DateTimeField(default=datetime.datetime.now)
     server_order = CharField(default="")
+    preparing = BooleanField(default=False)
 
     class Meta:
         table_name = "users"
@@ -197,6 +198,19 @@ class helper_users:
     def set_support_path(user_id, support_path):
         Users.update(support_logs = support_path).where(Users.user_id == user_id).execute()
 
+    @staticmethod
+    def set_prepare(user_id):
+        Users.update(preparing = True).where(Users.user_id == user_id).execute()
+
+    @staticmethod
+    def stop_prepare(user_id):
+        Users.update(preparing = False).where(Users.user_id == user_id).execute()
+
+    @staticmethod
+    def clear_support_status():
+        #pylint: disable=singleton-comparison
+        Users.update(preparing = False).where(Users.preparing == True).execute()
+
     @staticmethod
     def user_id_exists(user_id):
         if not users_helper.get_user(user_id):

app/classes/shared/helpers.py

@@ -424,6 +424,30 @@ class Helpers:
         now = datetime.now()
         return now.strftime("%m/%d/%Y, %H:%M:%S")
 
+    @staticmethod
+    def calc_percent(source_path, dest_path):
+        #calculates percentable of zip from drive. Not with compression. For backups and support logs
+        source_size = 0
+        files_count = 0
+        for path, _dirs, files in os.walk(source_path):
+            for f in files:
+                fp = os.path.join(path, f)
+                source_size += os.stat(fp).st_size
+                files_count += 1
+        dest_size = os.path.getsize(str(dest_path))
+        percent = round((dest_size/source_size) * 100, 1)
+        if percent >= 0:
+            results = {
+                "percent": percent,
+                "total_files": files_count
+            }
+        else:
+            results = {
+                "percent": 0,
+                "total_files": 0
+            }
+        return results
+
     @staticmethod
     def check_file_exists(path: str):
         logger.debug(f'Looking for path: {path}')

app/classes/shared/main_controller.py

@@ -28,6 +28,9 @@ from app.classes.web.websocket_helper import websocket_helper
 
 try:
     from peewee import DoesNotExist
+    #TZLocal is set as a hidden import on win pipeline
+    from tzlocal import get_localzone
+    from apscheduler.schedulers.background import BackgroundScheduler
 
 except ModuleNotFoundError as err:
     helper.auto_installer_fix(err)
@@ -45,6 +48,9 @@ class Controller:
         self.server_perms = Server_Perms_Controller()
         self.servers = Servers_Controller()
         self.users = Users_Controller()
+        tz = get_localzone()
+        self.support_scheduler = BackgroundScheduler(timezone=str(tz))
+        self.support_scheduler.start()
 
     def check_server_loaded(self, server_id_to_check: int):
 
@@ -128,6 +134,9 @@ class Controller:
 
 
     def package_support_logs(self, exec_user):
+        if exec_user['preparing']:
+            return
+        self.users.set_prepare(exec_user['user_id'])
         #pausing so on screen notifications can run for user
         time.sleep(7)
         websocket_helper.broadcast_user(exec_user['user_id'], 'notification', 'Preparing your support logs')
@@ -161,14 +170,22 @@ class Controller:
         #Copy crafty logs to archive dir
         full_log_name = os.path.join(crafty_path, 'logs')
         file_helper.copy_dir(os.path.join(self.project_root, 'logs'), full_log_name)
+        self.support_scheduler.add_job(self.log_status, 'interval', seconds=1, id="logs_"+str(exec_user['user_id']), args = [full_temp,
+                tempZipStorage +'.zip', exec_user])
         file_helper.make_archive(tempZipStorage, tempDir)
 
+        if len(websocket_helper.clients) > 0:
+            websocket_helper.broadcast_user(exec_user['user_id'], 'support_status_update', helper.calc_percent(full_temp, tempZipStorage +'.zip'))
+
         tempZipStorage += '.zip'
         websocket_helper.broadcast_user(exec_user['user_id'], 'send_logs_bootbox', {
                 })
 
         self.users.set_support_path(exec_user['user_id'], tempZipStorage)
 
+        self.users.stop_prepare(exec_user['user_id'])
+        self.support_scheduler.remove_job('logs_'+str(exec_user["user_id"]))
+
     @staticmethod
     def add_system_user():
         helper_users.add_user("system", helper.random_string_generator(64), "default@example.com", False, False)
@@ -190,6 +207,22 @@ class Controller:
         else:
             svr.stop_crash_detection()
 
+    def log_status(self, source_path, dest_path, exec_user):
+        results = helper.calc_percent(source_path, dest_path)
+        self.log_stats = results
+
+        if len(websocket_helper.clients) > 0:
+            websocket_helper.broadcast_user(exec_user['user_id'], 'support_status_update', results)
+
+    def send_log_status(self):
+        try:
+            return self.log_stats
+        except:
+            return {
+                'percent': 0,
+                'total_files': 0
+            }
+
     def get_server_obj(self, server_id: Union[str, int]) -> Union[bool, Server]:
         for s in self.servers_list:
             if str(s['server_id']) == str(server_id):
@@ -603,3 +636,7 @@ class Controller:
     @staticmethod
     def clear_unexecuted_commands():
         helpers_management.clear_unexecuted_commands()
+
+    @staticmethod
+    def clear_support_status():
+        helper_users.clear_support_status()

app/classes/shared/server.py

@@ -594,6 +594,7 @@ class Server:
         if not self.is_backingup:
             try:
                 backup_thread.start()
+                self.is_backingup = True
             except Exception as ex:
                 logger.error(f"Failed to start backup: {ex}")
                 return False
@@ -603,20 +604,34 @@ class Server:
         logger.info(f"Backup Thread started for server {self.settings['server_name']}.")
 
     def a_backup_server(self):
+        if len(websocket_helper.clients) > 0:
+            websocket_helper.broadcast_page_params(
+            '/panel/server_detail',
+            {
+                'id': str(self.server_id)
+            },
+            'backup_reload',
+            {
+                "percent": 0,
+                "total_files": 0
+            }
+        )
         logger.info(f"Starting server {self.name} (ID {self.server_id}) backup")
         server_users = server_permissions.get_server_user_list(self.server_id)
         for user in server_users:
             websocket_helper.broadcast_user(user, 'notification', translation.translate('notify',
             'backupStarted', users_helper.get_user_lang_by_id(user)).format(self.name))
         time.sleep(3)
-        self.is_backingup = True
         conf = management_helper.get_backup_config(self.server_id)
+        helper.ensure_dir_exists(self.settings['backup_path'])
         try:
             backup_filename = f"{self.settings['backup_path']}/{datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}"
             logger.info(f"Creating backup of server '{self.settings['server_name']}'" +
                         f" (ID#{self.server_id}, path={self.server_path}) at '{backup_filename}'")
 
             tempDir = tempfile.mkdtemp()
+            self.server_scheduler.add_job(self.backup_status, 'interval', seconds=1, id="backup_"+str(self.server_id), args = [tempDir+'/',
+                                backup_filename+'.zip'])
             # pylint: disable=unexpected-keyword-arg
             file_helper.copy_dir(self.server_path, tempDir, dirs_exist_ok=True)
             excluded_dirs = management_helper.get_excluded_backup_dirs(self.server_id)
@@ -650,6 +665,21 @@ class Server:
             self.is_backingup = False
             file_helper.del_dirs(tempDir)
             logger.info(f"Backup of server: {self.name} completed")
+            self.server_scheduler.remove_job("backup_"+str(self.server_id))
+            results = {
+                "percent": 100,
+                "total_files": 0,
+                "current_file": 0
+            }
+            if len(websocket_helper.clients) > 0:
+                websocket_helper.broadcast_page_params(
+                '/panel/server_detail',
+                {
+                    'id': str(self.server_id)
+                },
+                'backup_status',
+                results
+            )
             server_users = server_permissions.get_server_user_list(self.server_id)
             for user in server_users:
                 websocket_helper.broadcast_user(user, 'notification', translation.translate('notify', 'backupComplete',
@@ -658,9 +688,46 @@ class Server:
             return
         except:
             logger.exception(f"Failed to create backup of server {self.name} (ID {self.server_id})")
+            self.server_scheduler.remove_job("backup_"+str(self.server_id))
+            results = {
+                "percent": 100,
+                "total_files": 0,
+                "current_file": 0
+            }
+            if len(websocket_helper.clients) > 0:
+                websocket_helper.broadcast_page_params(
+                '/panel/server_detail',
+                {
+                    'id': str(self.server_id)
+                },
+                'backup_status',
+                results
+            )
             self.is_backingup = False
             return
 
+    def backup_status(self, source_path, dest_path):
+        results = helper.calc_percent(source_path, dest_path)
+        self.backup_stats = results
+        if len(websocket_helper.clients) > 0:
+            websocket_helper.broadcast_page_params(
+            '/panel/server_detail',
+            {
+                'id': str(self.server_id)
+            },
+            'backup_status',
+            results
+        )
+
+    def send_backup_status(self):
+        try:
+            return self.backup_stats
+        except:
+            return {
+                'percent': 0,
+                'total_files': 0
+            }
+
     def list_backups(self):
         if self.settings['backup_path']:
             if helper.check_path_exists(helper.get_os_understandable_path(self.settings['backup_path'])):

app/classes/web/ajax_handler.py

@@ -309,6 +309,19 @@ class AjaxHandler(BaseHandler):
             self.controller.users.update_server_order(exec_user['user_id'], bleach.clean(self.get_argument('order')))
             return
 
+        elif page == "backup_now":
+            server_id = self.get_argument('id', None)
+            if server_id is None:
+                return
+
+            server = self.controller.get_server_obj(server_id)
+            self.controller.management.add_to_audit_log_raw(
+                self.controller.users.get_user_by_id(exec_user['user_id'])['username'], exec_user['user_id'], server_id,
+                f"Backup now executed for server {server_id} ",
+                source_ip=self.get_remote_ip())
+
+            server.backup_server()
+
         elif page == "clear_comms":
             if exec_user['superuser']:
                 self.controller.clear_unexecuted_commands()

app/classes/web/panel_handler.py

@@ -184,6 +184,7 @@ class PanelHandler(BaseHandler):
         # pylint: disable=unused-variable
         api_key, token_data, exec_user = self.current_user
         superuser = exec_user['superuser']
+        preparing = exec_user['preparing']
         if api_key is not None:
             superuser = superuser and api_key.superuser
 
@@ -254,6 +255,7 @@ class PanelHandler(BaseHandler):
             'lang': self.controller.users.get_user_lang_by_id(exec_user["user_id"]),
             'lang_page': helper.getLangPage(self.controller.users.get_user_lang_by_id(exec_user["user_id"])),
             'super_user': superuser,
+            'preparing': preparing,
             'api_key': {
                 'name': api_key.name,
                 'created': api_key.created,
@@ -466,6 +468,8 @@ class PanelHandler(BaseHandler):
                 page_data['backup_config'] = self.controller.management.get_backup_config(server_id)
                 exclusions = []
                 page_data['exclusions'] = self.controller.management.get_excluded_backup_dirs(server_id)
+                page_data['backing_up'] = self.controller.get_server_obj(server_id).is_backingup
+                page_data['backup_stats'] = self.controller.get_server_obj(server_id).send_backup_status()
                 #makes it so relative path is the only thing shown
                 for file in page_data['exclusions']:
                     if helper.is_os_windows():
@@ -526,20 +530,6 @@ class PanelHandler(BaseHandler):
 
             self.redirect(f"/panel/server_detail?id={server_id}&subpage=backup")
 
-        elif page == 'backup_now':
-            server_id = self.check_server_id()
-            if server_id is None:
-                return
-
-            server = self.controller.get_server_obj(server_id)
-            management_helper.add_to_audit_log_raw(
-                self.controller.users.get_user_by_id(exec_user['user_id'])['username'], exec_user['user_id'], server_id,
-                f"Backup now executed for server {server_id} ",
-                source_ip=self.get_remote_ip())
-
-            server.backup_server()
-            self.redirect(f"/panel/server_detail?id={server_id}&subpage=backup")
-
         elif page == 'panel_config':
             auth_servers = {}
             auth_role_servers = {}

app/frontend/templates/base.html

@@ -214,9 +214,14 @@
       };
       wsInternal.onerror = function (errorEvent) {
         console.error('WebSocket Error', errorEvent);
+        warn('WebSockets are required for Crafty to work. This websocket connection has been closed. Are you using a reverse proxy? See our'+
+              ' documentation for details')
+
       };
       wsInternal.onclose = function (closeEvent) {
         console.log('Closed WebSocket', closeEvent);
+        warn('WebSockets are required for Crafty to work. This websocket connection has been closed. Are you using a reverse proxy? See our'+
+              ' documentation for details')
       };
 
 
@@ -262,6 +267,17 @@
         })
       });
     }
+    if (webSocket) {
+      webSocket.on('support_status_update', function (logs) {
+        if(logs.percent >= 100){
+          document.getElementById('logs_progress_bar').innerHTML = '100%';
+          document.getElementById('logs_progress_bar').style.width = '100%';
+        }else{
+          document.getElementById('logs_progress_bar').innerHTML = logs.percent +'%';
+          document.getElementById('logs_progress_bar').style.width = logs.percent + '%';
+        }
+      });
+    }
     if (webSocket) {
       webSocket.on('send_logs_bootbox', function (server_id) {
         var x = document.querySelector('.bootbox');

app/frontend/templates/notify.html

@@ -32,7 +32,14 @@
           {% end %}
           <p class="font-weight-light text-muted mb-0">Email: {{ data['user_data']['email'] }}</p>
         </div>
+        {% if data['preparing'] %}
+        <span class="dropdown-item" id="support_progress"><i class="dropdown-item-icon mdi mdi-download-outline text-primary"></i>{{ translate('notify', 'supportLogs', data['lang']) }}<br><br></span>
+        <span class="dropdown-item" id="support_progress"><div class="support_progress" style="height: 15px; width: 100%;">
+          <div class="progress-bar progress-bar-striped progress-bar-animated" id="logs_progress_bar" role="progressbar" style="width:0%;" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">0%</div>
+        </div></span>
+        {% else %}
         <a class="dropdown-item" id="support_logs" ><i class="dropdown-item-icon mdi mdi-download-outline text-primary"></i>{{ translate('notify', 'supportLogs', data['lang']) }}</i></a>
+        {% end %}
         {% if data['superuser'] %}
         <a class="dropdown-item" href="/panel/activity_logs"><i class="dropdown-item-icon mdi mdi-calendar-check-outline text-primary"></i>{{ translate('notify', 'activityLog', data['lang']) }}</a>
         {% end %}

app/frontend/templates/panel/server_backup.html

@@ -42,9 +42,20 @@
                       <input type="hidden" name="id" value="{{ data['server_stats']['server_id']['server_id'] }}">
                       <input type="hidden" name="subpage" value="backup">
 
-                      <div class="form-group">
-                        <a href="/panel/backup_now?id={{ data['server_stats']['server_id']['server_id'] }}" class="btn btn-primary" onclick="backup_started()">{{ translate('serverBackups', 'backupNow', data['lang']) }}</a>
+
+                      {% if data['backing_up'] %}
+                      <div class="progress" style="height: 15px;">
+                        <div class="progress-bar progress-bar-striped progress-bar-animated" id="backup_progress_bar" role="progressbar" style="width:{{data['backup_stats']['percent']}}%;" aria-valuenow="{{data['backup_stats']['percent']}}" aria-valuemin="0" aria-valuemax="100">{{ data['backup_stats']['percent'] }}%</div>
                       </div>
+                      <p>Backing up <span id="total_files">{{data['backup_stats']['total_files']}}</span> Files</p>
+                      {% end %}
+
+                      <br>
+                      {% if not data['backing_up'] %}
+                      <div id="backup_button" class="form-group">
+                        <button class="btn btn-primary" id="backup_now_button">{{ translate('serverBackups', 'backupNow', data['lang']) }}</button>
+                      </div>
+                      {% end %}
                       <div class="form-group">
                         {% if data['super_user'] %}
                         <label for="server_name">{{ translate('serverBackups', 'storageLocation', data['lang']) }} <small class="text-muted ml-1"> - {{ translate('serverBackups', 'storageLocationDesc', data['lang']) }}</small> </label>
@@ -224,12 +235,21 @@ const server_id = new URLSearchParams(document.location.search).get('id')
     return r ? r[1] : undefined;
   }
 
-  function backup_started(time='5-10') {
-    bootbox.alert({
+  function backup_started() {
+    var token = getCookie("_xsrf")
+    document.getElementById('backup_button').style.visibility = 'hidden';
+    var dialog = bootbox.dialog({
       message: "{{ translate('serverBackups', 'backupTask', data['lang']) }}",
-      backdrop: true
-    });
-  }
+      closeButton: false
+  });
+  $.ajax({
+    type: "POST",
+    headers: {'X-XSRFToken': token},
+    url: '/ajax/backup_now?id='+server_id,
+    success: function(data) {
+    },
+  });
+}
 
   function del_backup(filename, id){
     var token = getCookie("_xsrf")
@@ -350,6 +370,9 @@ const server_id = new URLSearchParams(document.location.search).get('id')
         }
       });
     });
+    $( "#backup_now_button" ).click(function() {
+      backup_started();
+    });
 
   });
 
@@ -402,6 +425,26 @@ document.getElementById("modal-cancel").addEventListener("click", function(){
     }, 5000);
     });
   }
+  if (webSocket) {
+    webSocket.on('backup_status', function (backup) {
+      if(backup.percent >= 100){
+        document.getElementById('backup_progress_bar').innerHTML = '100%';
+        document.getElementById('backup_progress_bar').style.width = '100%';
+        setTimeout(function(){
+          window.location.reload(1);
+       }, 5000);
+      }else{
+        document.getElementById('backup_progress_bar').innerHTML = backup.percent +'%';
+        document.getElementById('backup_progress_bar').style.width = backup.percent + '%';
+        document.getElementById('total_files').innerHTML = backup.total_files;
+      }
+    });
+  }
+  if (webSocket) {
+    webSocket.on('backup_reload', function (backup) {
+      location.reload()
+    });
+  }
 
   function getTreeView(path) {
     path = path

app/migrations/20220312_support_log_status.py

@@ -0,0 +1,16 @@
+# Generated by database migrator
+import peewee
+
+def migrate(migrator, database, **kwargs):
+    migrator.add_columns('users', preparing=peewee.BooleanField(default=False))
+    """
+    Write your migrations here.
+    """
+
+
+
+def rollback(migrator, database, **kwargs):
+    migrator.drop_columns('users', ['preparing'])
+    """
+    Write your rollback migrations here.
+    """

main.py

@@ -148,6 +148,7 @@ if __name__ == '__main__':
     project_root = os.path.dirname(__file__)
     controller.set_project_root(project_root)
     controller.clear_unexecuted_commands()
+    controller.clear_support_status()
 
     def sigterm_handler(*sig):
         print() # for newline