From 4bac56e84aba83ccc392267d69014d680f6a20ed Mon Sep 17 00:00:00 2001 From: luukas Date: Tue, 10 Aug 2021 23:17:56 +0300 Subject: [PATCH] Use stdout for virtual terminal. WebSockets seem to be "laggy". --- app/classes/shared/helpers.py | 6 ++ app/classes/shared/server.py | 62 ++++++++++++++++++- app/classes/shared/tasks.py | 8 +-- app/classes/web/ajax_handler.py | 6 +- app/classes/web/websocket_handler.py | 15 +++-- app/classes/web/websocket_helper.py | 50 ++++++++++++--- app/config/config.json | 4 +- app/config/logging.json | 2 +- app/frontend/templates/base.html | 5 +- app/frontend/templates/panel/server_term.html | 16 ++++- 10 files changed, 146 insertions(+), 28 deletions(-) diff --git a/app/classes/shared/helpers.py b/app/classes/shared/helpers.py index 176c98db..2e675e3a 100644 --- a/app/classes/shared/helpers.py +++ b/app/classes/shared/helpers.py @@ -562,4 +562,10 @@ class Helpers: os.path.relpath(os.path.join(root, file), os.path.join(path, '..'))) + @staticmethod + def remove_prefix(text, prefix): + if text.startswith(prefix): + return text[len(prefix):] + return text + helper = Helpers() diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index ab6f6372..c4901125 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -10,11 +10,13 @@ import threading import schedule import logging.config import zipfile +import html from app.classes.shared.helpers import helper from app.classes.shared.console import console from app.classes.shared.models import db_helper, Servers +from app.classes.web.websocket_helper import websocket_helper logger = logging.getLogger(__name__) @@ -27,6 +29,58 @@ except ModuleNotFoundError as e: console.critical("Import Error: Unable to load {} module".format(e.name)) sys.exit(1) +class ServerOutBuf: + lines = {} + def __init__(self, p, server_id): + self.p = p + self.server_id = str(server_id) + # Buffers text for virtual_terminal_lines config number of lines + self.max_lines = helper.get_setting('virtual_terminal_lines') + self.line_buffer = '' + ServerOutBuf.lines[self.server_id] = [] + + def check(self): + while self.p.isalive(): + char = self.p.read(1) + if char == os.linesep: + ServerOutBuf.lines[self.server_id].append(self.line_buffer) + self.new_line_handler(self.line_buffer) + self.line_buffer = '' + # Limit list length to self.max_lines: + if len(ServerOutBuf.lines[self.server_id]) > self.max_lines: + ServerOutBuf.lines[self.server_id].pop(0) + else: + self.line_buffer += char + + def new_line_handler(self, new_line): + console.debug('New line: {}'.format(new_line)) + + highlighted = helper.log_colors(html.escape(new_line)) + + print('broadcasting new vterm line') + + websocket_helper.broadcast_page_params( + '/panel/server_detail', + { + 'id': self.server_id + }, + 'notification', + 'test test test' + ) + + # TODO: Do not send data to clients who do not have permission to view this server's console + websocket_helper.broadcast_page_params( + '/panel/server_detail', + { + 'id': self.server_id + }, + 'vterm_new_line', + { + 'line': highlighted + '
', + 'server_id': self.server_id + } + ) + class Server: @@ -127,7 +181,13 @@ class Server: logger.info("Linux Detected") logger.info("Starting server in {p} with command: {c}".format(p=self.server_path, c=self.server_command)) - self.process = pexpect.spawn(self.server_command, cwd=self.server_path, timeout=None, encoding=None) + + self.process = pexpect.spawn(self.server_command, cwd=self.server_path, timeout=None, encoding='utf-8') + out_buf = ServerOutBuf(self.process, self.server_id) + + console.cyan('Start vterm listener') + threading.Thread(target=out_buf.check, daemon=True).start() + self.is_crashed = False self.start_time = str(datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')) diff --git a/app/classes/shared/tasks.py b/app/classes/shared/tasks.py index 3a23eeaf..69c98e93 100644 --- a/app/classes/shared/tasks.py +++ b/app/classes/shared/tasks.py @@ -196,8 +196,9 @@ class TasksManager: host_stats = db_helper.get_latest_hosts_stats() if len(websocket_helper.clients) > 0: + print('there are clients') # There are clients - websocket_helper.broadcast('update_host_stats', { + websocket_helper.broadcast_page('/panel/dashboard', 'update_host_stats', { 'cpu_usage': host_stats.get('cpu_usage'), 'cpu_cores': host_stats.get('cpu_cores'), 'cpu_cur_freq': host_stats.get('cpu_cur_freq'), @@ -205,10 +206,7 @@ class TasksManager: 'mem_percent': host_stats.get('mem_percent'), 'mem_usage': host_stats.get('mem_usage') }) - time.sleep(4) - else: - # Stats are same - time.sleep(8) + time.sleep(4) def log_watcher(self): helper.check_for_old_logs(db_helper) diff --git a/app/classes/web/ajax_handler.py b/app/classes/web/ajax_handler.py index 4f98d98f..35bfb990 100644 --- a/app/classes/web/ajax_handler.py +++ b/app/classes/web/ajax_handler.py @@ -12,6 +12,7 @@ from app.classes.shared.models import Users, installer from app.classes.web.base_handler import BaseHandler from app.classes.shared.models import db_helper from app.classes.shared.helpers import helper +from app.classes.shared.server import ServerOutBuf logger = logging.getLogger(__name__) @@ -56,16 +57,17 @@ class AjaxHandler(BaseHandler): if not server_data: logger.warning("Server Data not found in server_log ajax call") self.redirect("/panel/error?error=Server ID Not Found") + return if not server_data['log_path']: logger.warning("Log path not found in server_log ajax call ({})".format(server_id)) if full_log: log_lines = helper.get_setting('max_log_lines') + data = helper.tail_file(server_data['log_path'], log_lines) else: - log_lines = helper.get_setting('virtual_terminal_lines') + data = ServerOutBuf.lines.get(server_id, []) - data = helper.tail_file(server_data['log_path'], log_lines) for d in data: try: diff --git a/app/classes/web/websocket_handler.py b/app/classes/web/websocket_handler.py index 107acdc5..3c83dbc0 100644 --- a/app/classes/web/websocket_handler.py +++ b/app/classes/web/websocket_handler.py @@ -1,9 +1,10 @@ import json import logging +from urllib.parse import parse_qsl import tornado.websocket -from app.classes.shared.console import console from app.classes.shared.models import Users, db_helper +from app.classes.shared.helpers import helper from app.classes.web.websocket_helper import websocket_helper logger = logging.getLogger(__name__) @@ -35,6 +36,7 @@ class SocketHandler(tornado.websocket.WebSocketHandler): def open(self): + logger.debug('Checking WebSocket authentication') if self.check_auth(): self.handle() else: @@ -42,10 +44,15 @@ class SocketHandler(tornado.websocket.WebSocketHandler): self.close() db_helper.add_to_audit_log_raw('unknown', 0, 0, 'Someone tried to connect via WebSocket without proper authentication', self.get_remote_ip()) websocket_helper.broadcast('notification', 'Someone tried to connect via WebSocket without proper authentication') + logger.warning('Someone tried to connect via WebSocket without proper authentication') def handle(self): - - websocket_helper.addClient(self) + self.page = self.get_query_argument('page') + self.page_query_params = dict(parse_qsl(helper.remove_prefix( + self.get_query_argument('page_query_params'), + '?' + ))) + websocket_helper.add_client(self) logger.debug('Opened WebSocket connection') # websocket_helper.broadcast('notification', 'New client connected') @@ -56,7 +63,7 @@ class SocketHandler(tornado.websocket.WebSocketHandler): logger.debug('Event Type: {}, Data: {}'.format(message['event'], message['data'])) def on_close(self): - websocket_helper.removeClient(self) + websocket_helper.remove_client(self) logger.debug('Closed WebSocket connection') # websocket_helper.broadcast('notification', 'Client disconnected') diff --git a/app/classes/web/websocket_helper.py b/app/classes/web/websocket_helper.py index 37a85cb4..5fdd72fb 100644 --- a/app/classes/web/websocket_helper.py +++ b/app/classes/web/websocket_helper.py @@ -6,25 +6,59 @@ from app.classes.shared.console import console logger = logging.getLogger(__name__) class WebSocketHelper: - clients = set() + def __init__(self): + self.clients = set() - def addClient(self, client): + def add_client(self, client): self.clients.add(client) - def removeClient(self, client): - self.clients.add(client) + def remove_client(self, client): + self.clients.remove(client) - def send_message(self, client, event_type, data): + def send_message(self, client, event_type: str, data): if client.check_auth(): message = str(json.dumps({'event': event_type, 'data': data})) client.write_message(message) - def broadcast(self, event_type, data): - logger.debug('Sending: ' + str(json.dumps({'event': event_type, 'data': data}))) + def broadcast(self, event_type: str, data): + logger.debug('Sending to {} clients: {}'.format(len(self.clients), json.dumps({'event': event_type, 'data': data}))) for client in self.clients: try: self.send_message(client, event_type, data) - except: + except Exception: + pass + + def broadcast_page(self, page: str, event_type: str, data): + def filter_fn(client): + return client.page == page + + clients = list(filter(filter_fn, self.clients)) + + logger.debug('Sending to {} out of {} clients: {}'.format(len(clients), len(self.clients), json.dumps({'event': event_type, 'data': data}))) + + for client in clients: + try: + self.send_message(client, event_type, data) + except Exception: + pass + + def broadcast_page_params(self, page: str, params: dict, event_type: str, data): + def filter_fn(client): + if client.page != page: + return False + for key, param in params.items(): + if param != client.page_query_params.get(key, None): + return False + return True + + clients = list(filter(filter_fn, self.clients)) + + logger.debug('Sending to {} out of {} clients: {}'.format(len(clients), len(self.clients), json.dumps({'event': event_type, 'data': data}))) + + for client in clients: + try: + self.send_message(client, event_type, data) + except Exception: pass def disconnect_all(self): diff --git a/app/config/config.json b/app/config/config.json index f6b866f3..693c1440 100644 --- a/app/config/config.json +++ b/app/config/config.json @@ -10,7 +10,7 @@ "stats_update_frequency": 30, "delete_default_json": false, "show_contribute_link": true, - "virtual_terminal_lines": 10, + "virtual_terminal_lines": 30, "max_log_lines": 700, "keywords": ["help", "chunk"] -} \ No newline at end of file +} diff --git a/app/config/logging.json b/app/config/logging.json index 8c524b01..c6ccd7b1 100644 --- a/app/config/logging.json +++ b/app/config/logging.json @@ -8,7 +8,7 @@ "tornado_access": { "format": "%(asctime)s - [Tornado] - [Access] - %(levelname)s - %(message)s" }, - "schedule": { + "schedule": { "format": "%(asctime)s - [Schedules] - %(levelname)s - %(message)s" } }, diff --git a/app/frontend/templates/base.html b/app/frontend/templates/base.html index 2a780c64..ad2cb9a5 100644 --- a/app/frontend/templates/base.html +++ b/app/frontend/templates/base.html @@ -173,8 +173,9 @@ let listenEvents = []; try { - - var wsInternal = new WebSocket('wss://' + location.host + '/ws'); + pageQueryParams = 'page_query_params=' + encodeURIComponent(location.search) + page = 'page=' + encodeURIComponent(location.pathname) + var wsInternal = new WebSocket('wss://' + location.host + '/ws?' + page + '&' + pageQueryParams); wsInternal.onopen = function() { console.log('opened WebSocket connection:', wsInternal) }; diff --git a/app/frontend/templates/panel/server_term.html b/app/frontend/templates/panel/server_term.html index 196c56da..92c7e54c 100644 --- a/app/frontend/templates/panel/server_term.html +++ b/app/frontend/templates/panel/server_term.html @@ -161,6 +161,12 @@ } } + function new_line_handler(data) { + if (server_id === data.server_id) { + $('#virt_console').append(data.line) + } + } + //used to get cookies from browser - this is part of tornados xsrf protection - it's for extra security function getCookie(name) { var r = document.cookie.match("\\b" + name + "=([^;]*)\\b"); @@ -171,9 +177,13 @@ console.log( "ready!" ); get_server_log() - setInterval(function(){ - get_server_log() // this will run after every 5 seconds - }, 1500); + if (webSocket) { + webSocket.on('vterm_new_line', new_line_handler) + } else { + setInterval(function(){ + get_server_log() // this will run after every 5 seconds + }, 1500); + } }); $('#server_command').on('keydown', function (e) {