From 375df0c27da02aea113685eb31a47edf760afbec Mon Sep 17 00:00:00 2001 From: amcmanu3 Date: Fri, 9 Sep 2022 15:00:08 -0400 Subject: [PATCH 1/5] Only check for gravatar if internet --- app/classes/web/panel_handler.py | 6 +++++- app/classes/web/routes/api/users/user/pfp.py | 6 +++++- app/classes/web/server_handler.py | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/app/classes/web/panel_handler.py b/app/classes/web/panel_handler.py index 44844bbf..0c858772 100644 --- a/app/classes/web/panel_handler.py +++ b/app/classes/web/panel_handler.py @@ -338,7 +338,11 @@ class PanelHandler(BaseHandler): rating = "g" # Get grvatar hash for profile pictures - if exec_user["email"] != "default@example.com" or "": + if ( + self.helper.check_internet() + and exec_user["email"] != "default@example.com" + and exec_user["email"] == "" + ): gravatar = libgravatar.Gravatar( libgravatar.sanitize_email(exec_user["email"]) ) diff --git a/app/classes/web/routes/api/users/user/pfp.py b/app/classes/web/routes/api/users/user/pfp.py index a4f0a480..0e68027c 100644 --- a/app/classes/web/routes/api/users/user/pfp.py +++ b/app/classes/web/routes/api/users/user/pfp.py @@ -28,7 +28,11 @@ class ApiUsersUserPfpHandler(BaseApiHandler): rating = "g" # Get grvatar hash for profile pictures - if user["email"] != "default@example.com" or "": + if ( + self.helper.check_internet() + and user["email"] != "default@example.com" + and user["email"] == "" + ): gravatar = libgravatar.Gravatar(libgravatar.sanitize_email(user["email"])) url = gravatar.get_image( size=80, diff --git a/app/classes/web/server_handler.py b/app/classes/web/server_handler.py index 1c1d6fc6..5b5b2c49 100644 --- a/app/classes/web/server_handler.py +++ b/app/classes/web/server_handler.py @@ -138,7 +138,11 @@ class ServerHandler(BaseHandler): else: rating = "g" - if exec_user["email"] != "default@example.com" or "": + if ( + self.helper.check_internet() + and exec_user["email"] != "default@example.com" + and exec_user["email"] == "" + ): gravatar = libgravatar.Gravatar( libgravatar.sanitize_email(exec_user["email"]) ) From 2b35da71fcafaf14685ad3ab6bad6999cb19304c Mon Sep 17 00:00:00 2001 From: amcmanu3 Date: Fri, 9 Sep 2022 15:04:45 -0400 Subject: [PATCH 2/5] Fix logic with gravatar email --- app/classes/web/panel_handler.py | 6 +++--- app/classes/web/routes/api/users/user/pfp.py | 6 +++--- app/classes/web/server_handler.py | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/classes/web/panel_handler.py b/app/classes/web/panel_handler.py index 0c858772..2192a55b 100644 --- a/app/classes/web/panel_handler.py +++ b/app/classes/web/panel_handler.py @@ -339,9 +339,9 @@ class PanelHandler(BaseHandler): # Get grvatar hash for profile pictures if ( - self.helper.check_internet() - and exec_user["email"] != "default@example.com" - and exec_user["email"] == "" + not self.helper.check_internet() + or exec_user["email"] != "default@example.com" + or exec_user["email"] != "" ): gravatar = libgravatar.Gravatar( libgravatar.sanitize_email(exec_user["email"]) diff --git a/app/classes/web/routes/api/users/user/pfp.py b/app/classes/web/routes/api/users/user/pfp.py index 0e68027c..86e88dbc 100644 --- a/app/classes/web/routes/api/users/user/pfp.py +++ b/app/classes/web/routes/api/users/user/pfp.py @@ -29,9 +29,9 @@ class ApiUsersUserPfpHandler(BaseApiHandler): # Get grvatar hash for profile pictures if ( - self.helper.check_internet() - and user["email"] != "default@example.com" - and user["email"] == "" + not self.helper.check_internet() + or user["email"] != "default@example.com" + or user["email"] != "" ): gravatar = libgravatar.Gravatar(libgravatar.sanitize_email(user["email"])) url = gravatar.get_image( diff --git a/app/classes/web/server_handler.py b/app/classes/web/server_handler.py index 5b5b2c49..954c31ae 100644 --- a/app/classes/web/server_handler.py +++ b/app/classes/web/server_handler.py @@ -139,9 +139,9 @@ class ServerHandler(BaseHandler): rating = "g" if ( - self.helper.check_internet() - and exec_user["email"] != "default@example.com" - and exec_user["email"] == "" + not self.helper.check_internet() + or exec_user["email"] != "default@example.com" + or exec_user["email"] != "" ): gravatar = libgravatar.Gravatar( libgravatar.sanitize_email(exec_user["email"]) From 82c0452f552dfdc5e43169728876ad5e01dbb614 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 12 Sep 2022 15:34:47 -0400 Subject: [PATCH 3/5] Add pfp caching for users --- app/classes/controllers/users_controller.py | 12 +++++++ app/classes/models/users.py | 2 ++ app/classes/shared/helpers.py | 28 +++++++++++++++ app/classes/shared/tasks.py | 6 ++++ app/classes/web/panel_handler.py | 36 -------------------- app/classes/web/routes/api/users/user/pfp.py | 34 ++---------------- app/classes/web/server_handler.py | 34 ------------------ app/frontend/templates/notify.html | 14 ++++++-- app/migrations/20220912_user_pfp.py | 19 +++++++++++ 9 files changed, 80 insertions(+), 105 deletions(-) create mode 100644 app/migrations/20220912_user_pfp.py diff --git a/app/classes/controllers/users_controller.py b/app/classes/controllers/users_controller.py index 13b8fb4f..d7ce4393 100644 --- a/app/classes/controllers/users_controller.py +++ b/app/classes/controllers/users_controller.py @@ -147,14 +147,24 @@ class UsersController: return HelperServers.get_total_owned_servers(exec_user_id) def update_user(self, user_id: str, user_data=None, user_crafty_data=None): + # check if user crafty perms were updated if user_crafty_data is None: user_crafty_data = {} + # check if general user data was updated if user_data is None: user_data = {} + # get current user data base_data = HelperUsers.get_user(user_id) up_data = {} + # check if we updated user email. If so we update gravatar + if user_data["email"]: + pfp = self.helper.get_gravatar_image(user_data["email"]) + up_data["pfp"] = pfp + # create sets to store role data added_roles = set() removed_roles = set() + + # search for changes in user data for key in user_data: if key == "user_id": continue @@ -174,8 +184,10 @@ class UsersController: up_data["hints"] = user_data["hints"] elif base_data[key] != user_data[key]: up_data[key] = user_data[key] + # change last update for user up_data["last_update"] = self.helper.get_time_as_string() logger.debug(f"user: {user_data} +role:{added_roles} -role:{removed_roles}") + for role in added_roles: HelperUsers.get_or_create(user_id=user_id, role_id=role) permissions_mask = user_crafty_data.get("permissions_mask", "000") diff --git a/app/classes/models/users.py b/app/classes/models/users.py index 0d8e596b..a6e87316 100644 --- a/app/classes/models/users.py +++ b/app/classes/models/users.py @@ -42,6 +42,7 @@ class Users(BaseModel): preparing = BooleanField(default=False) hints = BooleanField(default=True) manager = IntegerField(default=None, null=True) + pfp = CharField(default="/static/assets/images/faces-clipart/pic-3.png") class Meta: table_name = "users" @@ -220,6 +221,7 @@ class HelperUsers: Users.password: pw_enc, Users.email: email, Users.enabled: enabled, + Users.pfp: self.helper.get_gravatar_image(email), Users.superuser: superuser, Users.created: Helpers.get_time_as_string(), Users.manager: manager, diff --git a/app/classes/shared/helpers.py b/app/classes/shared/helpers.py index be278578..8d76131a 100644 --- a/app/classes/shared/helpers.py +++ b/app/classes/shared/helpers.py @@ -20,6 +20,7 @@ import itertools from datetime import datetime from socket import gethostname from contextlib import redirect_stderr, suppress +import libgravatar from packaging import version as pkg_version from app.classes.shared.null_writer import NullWriter @@ -658,6 +659,33 @@ class Helpers: return True return False + def get_gravatar_image(self, email): + profile_url = "/static/assets/images/faces-clipart/pic-3.png" + # http://en.gravatar.com/site/implement/images/#rating + if self.get_setting("allow_nsfw_profile_pictures"): + rating = "x" + else: + rating = "g" + + # Get grvatar hash for profile pictures + if not self.check_internet() or email != "default@example.com" or email != "": + gravatar = libgravatar.Gravatar(libgravatar.sanitize_email(email)) + url = gravatar.get_image( + size=80, + default="404", + force_default=False, + rating=rating, + filetype_extension=False, + use_ssl=True, + ) # + "?d=404" + try: + if requests.head(url).status_code != 404: + profile_url = url + except: + profile_url = "/static/assets/images/faces-clipart/pic-3.png" + + return profile_url + @staticmethod def get_file_contents(path: str, lines=100): diff --git a/app/classes/shared/tasks.py b/app/classes/shared/tasks.py index a654df6d..f3b5b24b 100644 --- a/app/classes/shared/tasks.py +++ b/app/classes/shared/tasks.py @@ -666,6 +666,12 @@ class TasksManager: logger.info( "No updates found! You are on the most up to date Crafty version." ) + logger.info("Refreshing Gravatar PFPs...") + for user in HelperUsers.get_all_users(): + if user.email: + HelperUsers.update_user( + user.id, {"pfp": self.helper.get_gravatar_image(user.email)} + ) def log_watcher(self): self.controller.servers.check_for_old_logs() diff --git a/app/classes/web/panel_handler.py b/app/classes/web/panel_handler.py index 2192a55b..59ec0abf 100644 --- a/app/classes/web/panel_handler.py +++ b/app/classes/web/panel_handler.py @@ -8,7 +8,6 @@ import logging import threading import shlex import bleach -import libgravatar import requests import tornado.web import tornado.escape @@ -331,41 +330,6 @@ class PanelHandler(BaseHandler): "superuser": superuser, } - # http://en.gravatar.com/site/implement/images/#rating - if self.helper.get_setting("allow_nsfw_profile_pictures"): - rating = "x" - else: - rating = "g" - - # Get grvatar hash for profile pictures - if ( - not self.helper.check_internet() - or exec_user["email"] != "default@example.com" - or exec_user["email"] != "" - ): - gravatar = libgravatar.Gravatar( - libgravatar.sanitize_email(exec_user["email"]) - ) - url = gravatar.get_image( - size=80, - default="404", - force_default=False, - rating=rating, - filetype_extension=False, - use_ssl=True, - ) # + "?d=404" - try: - if requests.head(url).status_code != 404: - profile_url = url - else: - profile_url = "/static/assets/images/faces-clipart/pic-3.png" - except: - profile_url = "/static/assets/images/faces-clipart/pic-3.png" - else: - profile_url = "/static/assets/images/faces-clipart/pic-3.png" - - page_data["user_image"] = profile_url - if page == "unauthorized": template = "panel/denied.html" diff --git a/app/classes/web/routes/api/users/user/pfp.py b/app/classes/web/routes/api/users/user/pfp.py index 86e88dbc..2bc10ba5 100644 --- a/app/classes/web/routes/api/users/user/pfp.py +++ b/app/classes/web/routes/api/users/user/pfp.py @@ -1,6 +1,4 @@ import logging -import libgravatar -import requests from app.classes.web.base_api_handler import BaseApiHandler logger = logging.getLogger(__name__) @@ -21,33 +19,5 @@ class ApiUsersUserPfpHandler(BaseApiHandler): f'User {auth_data[4]["user_id"]} is fetching the pfp for user {user_id}' ) - # http://en.gravatar.com/site/implement/images/#rating - if self.helper.get_setting("allow_nsfw_profile_pictures"): - rating = "x" - else: - rating = "g" - - # Get grvatar hash for profile pictures - if ( - not self.helper.check_internet() - or user["email"] != "default@example.com" - or user["email"] != "" - ): - gravatar = libgravatar.Gravatar(libgravatar.sanitize_email(user["email"])) - url = gravatar.get_image( - size=80, - default="404", - force_default=False, - rating=rating, - filetype_extension=False, - use_ssl=True, - ) - try: - requests.head(url).raise_for_status() - except requests.HTTPError as e: - logger.debug("Gravatar profile picture not found", exc_info=e) - else: - self.finish_json(200, {"status": "ok", "data": url}) - return - - self.finish_json(200, {"status": "ok", "data": None}) + self.finish_json(200, {"status": "ok", "data": user["pfp"]}) + return diff --git a/app/classes/web/server_handler.py b/app/classes/web/server_handler.py index 954c31ae..b2419905 100644 --- a/app/classes/web/server_handler.py +++ b/app/classes/web/server_handler.py @@ -5,8 +5,6 @@ import time import tornado.web import tornado.escape import bleach -import libgravatar -import requests from app.classes.models.crafty_permissions import EnumPermissionsCrafty from app.classes.shared.helpers import Helpers @@ -133,38 +131,6 @@ class ServerHandler(BaseHandler): "superuser": superuser, } - if self.helper.get_setting("allow_nsfw_profile_pictures"): - rating = "x" - else: - rating = "g" - - if ( - not self.helper.check_internet() - or exec_user["email"] != "default@example.com" - or exec_user["email"] != "" - ): - gravatar = libgravatar.Gravatar( - libgravatar.sanitize_email(exec_user["email"]) - ) - url = gravatar.get_image( - size=80, - default="404", - force_default=False, - rating=rating, - filetype_extension=False, - use_ssl=True, - ) # + "?d=404" - try: - if requests.head(url).status_code != 404: - profile_url = url - else: - profile_url = "/static/assets/images/faces-clipart/pic-3.png" - except: - profile_url = "/static/assets/images/faces-clipart/pic-3.png" - else: - profile_url = "/static/assets/images/faces-clipart/pic-3.png" - - page_data["user_image"] = profile_url if superuser: page_data["roles"] = list_roles diff --git a/app/frontend/templates/notify.html b/app/frontend/templates/notify.html index 07659b12..78f62409 100644 --- a/app/frontend/templates/notify.html +++ b/app/frontend/templates/notify.html @@ -18,10 +18,10 @@
  • + Profile image
    - Profile image + Profile image

    {{ data['user_data']['username'] }}

    Roles:

    {% for r in data['user_role'] %} @@ -46,4 +46,12 @@ {{ translate('notify', 'logout', data['lang']) }}
    • - \ No newline at end of file + + + \ No newline at end of file diff --git a/app/migrations/20220912_user_pfp.py b/app/migrations/20220912_user_pfp.py new file mode 100644 index 00000000..f70a28a8 --- /dev/null +++ b/app/migrations/20220912_user_pfp.py @@ -0,0 +1,19 @@ +# Generated by database migrator +import peewee + + +def migrate(migrator, database, **kwargs): + migrator.add_columns( + "users", + pfp=peewee.CharField(default="/static/assets/images/faces-clipart/pic-3.png"), + ) + """ + Write your migrations here. + """ + + +def rollback(migrator, database, **kwargs): + migrator.drop_columns("users", ["pfp"]) + """ + Write your rollback migrations here. + """ From 67d465b94ca952e8b4b840cef21c94b38b1d7ad4 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 12 Sep 2022 15:36:13 -0400 Subject: [PATCH 4/5] Add debug logging --- app/classes/shared/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/classes/shared/helpers.py b/app/classes/shared/helpers.py index 8d76131a..aa53132e 100644 --- a/app/classes/shared/helpers.py +++ b/app/classes/shared/helpers.py @@ -681,8 +681,8 @@ class Helpers: try: if requests.head(url).status_code != 404: profile_url = url - except: - profile_url = "/static/assets/images/faces-clipart/pic-3.png" + except Exception as e: + logger.debug(f"Could not pull resource from Gravatar with error {e}") return profile_url From be577317421fc691a90fed16193fb5125b97d73b Mon Sep 17 00:00:00 2001 From: Zedifus Date: Mon, 19 Sep 2022 22:20:00 +0100 Subject: [PATCH 5/5] Update changelog !459 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4a47f56..9f44f85d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ TBD - Fix logic issue when removing items from backup exclusions ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/453)) - Cleanup various JS errors ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/455)) - Temp fix for `&` issue in pathing and minecraft colour codes ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/457)) +- Cache Gravatar pfp's as to not query every page load ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/459)) ### Tweaks - Add button to scroll to bottom of vterm ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/454)) - Persist schedules and execution commands across backup restores ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/458))