From 6d9adee8d85c92bd4cd541a643c3cef722bfe4f1 Mon Sep 17 00:00:00 2001 From: zbeacon Date: Thu, 23 Apr 2020 08:31:22 +0300 Subject: [PATCH] Improvements in Remote Configuration feature. And if logging configured wrong, logs will be pushed to the ThingsBoard and only prints into console. --- .../gateway/tb_gateway_remote_configurator.py | 28 +++++++++++++++---- .../gateway/tb_gateway_service.py | 16 +++++++++-- thingsboard_gateway/gateway/tb_logger.py | 17 +++++++++++ 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/thingsboard_gateway/gateway/tb_gateway_remote_configurator.py b/thingsboard_gateway/gateway/tb_gateway_remote_configurator.py index 61f1f728..11811f05 100644 --- a/thingsboard_gateway/gateway/tb_gateway_remote_configurator.py +++ b/thingsboard_gateway/gateway/tb_gateway_remote_configurator.py @@ -12,13 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os import remove +from os import remove, linesep +from os.path import exists from re import findall from time import time, sleep from logging import getLogger from base64 import b64encode, b64decode from simplejson import dumps, loads, dump from yaml import safe_dump +from configparser import ConfigParser from thingsboard_gateway.gateway.tb_client import TBClient from thingsboard_gateway.tb_utility.tb_utility import TBUtility @@ -101,7 +103,7 @@ class RemoteConfigurator: self.__apply_storage_configuration() if self.__safe_apply_connection_configuration(): LOG.info("Remote configuration has been applied.") - with open(self.__gateway.get_config_path() + "tb_gateway.yaml", "w") as general_configuration_file: + with open(self.__gateway.get_config_path() + "tb_gateway.yaml", "w", encoding="UTF-8") as general_configuration_file: safe_dump(self.__new_general_configuration_file, general_configuration_file) self.__old_connectors_configs = {} self.__new_connectors_configs = {} @@ -114,7 +116,7 @@ class RemoteConfigurator: else: self.__update_logs_configuration() self.__old_general_configuration_file.pop("logs") - with open(self.__gateway.get_config_path() + "tb_gateway.yaml", "w") as general_configuration_file: + with open(self.__gateway.get_config_path() + "tb_gateway.yaml", "w", encoding="UTF-8") as general_configuration_file: safe_dump(self.__old_general_configuration_file, general_configuration_file) LOG.error("A remote general configuration applying has been failed.") self.__old_connectors_configs = {} @@ -167,7 +169,7 @@ class RemoteConfigurator: for connector_config_section in self.__new_connectors_configs[connector_type]: for connector_file in connector_config_section["config"]: connector_config = connector_config_section["config"][connector_file] - with open(self.__gateway.get_config_path() + connector_file, "w") as config_file: + with open(self.__gateway.get_config_path() + connector_file, "w", encoding="UTF-8") as config_file: dump(connector_config, config_file, sort_keys=True, indent=2) new_connectors_files.append(connector_file) LOG.debug("Saving new configuration for \"%s\" connector to file \"%s\"", connector_type, @@ -237,7 +239,7 @@ class RemoteConfigurator: def __get_current_logs_configuration(self): try: - with open(self.__gateway.get_config_path() + 'logs.conf', 'r') as logs: + with open(self.__gateway.get_config_path() + 'logs.conf', 'r', encoding="UTF-8") as logs: current_logs_configuration = logs.read() return current_logs_configuration except Exception as e: @@ -249,7 +251,20 @@ class RemoteConfigurator: LOG = getLogger('service') logs_conf_file_path = self.__gateway.get_config_path() + 'logs.conf' new_logging_level = findall(r'level=(.*)', self.__new_logs_configuration.replace("NONE", "NOTSET"))[-1] - with open(logs_conf_file_path, 'w') as logs: + new_logging_config = self.__new_logs_configuration.replace("NONE", "NOTSET").replace("\r\n", linesep) + logs_config = ConfigParser(allow_no_value=True) + logs_config.read_string(new_logging_config) + for section in logs_config: + if "handler_" in section and section != "handler_consoleHandler": + args = tuple(logs_config[section]["args"] + .replace('(', '') + .replace(')', '') + .split(', ')) + path = args[0][1:-1] + LOG.debug("Checking %s...", path) + if not exists(path): + raise FileNotFoundError + with open(logs_conf_file_path, 'w', encoding="UTF-8") as logs: logs.write(self.__new_logs_configuration.replace("NONE", "NOTSET")+"\r\n") self.__gateway.main_handler.setLevel(new_logging_level) self.__gateway.main_handler.setTarget(self.__gateway.remote_handler) @@ -259,5 +274,6 @@ class RemoteConfigurator: self.__gateway.remote_handler.activate(new_logging_level) LOG.debug("Logs configuration has been updated.") except Exception as e: + LOG.error("Remote logging configuration is wrong!") LOG.exception(e) diff --git a/thingsboard_gateway/gateway/tb_gateway_service.py b/thingsboard_gateway/gateway/tb_gateway_service.py index 010444ab..2ad45f1f 100644 --- a/thingsboard_gateway/gateway/tb_gateway_service.py +++ b/thingsboard_gateway/gateway/tb_gateway_service.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from sys import getsizeof, executable, argv +from sys import getsizeof, executable, argv, stdout from os import listdir, path, execv, pathsep, system from pkg_resources import get_distribution from time import time, sleep +import logging import logging.config import logging.handlers from queue import Queue @@ -33,6 +34,8 @@ from thingsboard_gateway.storage.memory_event_storage import MemoryEventStorage from thingsboard_gateway.storage.file_event_storage import FileEventStorage from thingsboard_gateway.gateway.tb_gateway_remote_configurator import RemoteConfigurator + + log = logging.getLogger('service') main_handler = logging.handlers.MemoryHandler(-1) @@ -45,7 +48,11 @@ class TBGatewayService: with open(config_file) as general_config: config = safe_load(general_config) self._config_dir = path.dirname(path.abspath(config_file)) + path.sep - logging.config.fileConfig(self._config_dir + "logs.conf") + logging_error = None + try: + logging.config.fileConfig(self._config_dir + "logs.conf", disable_existing_loggers=False) + except Exception as e: + logging_error = e global log log = logging.getLogger('service') log.info("Gateway starting...") @@ -62,6 +69,9 @@ class TBGatewayService: self.tb_client = TBClient(config["thingsboard"]) self.tb_client.connect() self.subscribe_to_required_topics() + if logging_error is not None: + self.tb_client.client.send_telemetry({"ts": time()*1000, "values": {"LOGS": "Logging loading exception, logs.conf is wrong: %s" % (str(logging_error), )}}) + TBLoggerHandler.set_default_handler() self.counter = 0 self.__rpc_reply_sent = False global main_handler @@ -227,7 +237,7 @@ class TBGatewayService: try: connector_class = TBUtility.check_and_import(connector["type"], self._default_connectors.get(connector["type"], connector.get("class"))) self._implemented_connectors[connector["type"]] = connector_class - with open(self._config_dir + connector['configuration'], 'r') as conf_file: + with open(self._config_dir + connector['configuration'], 'r', encoding="UTF-8") as conf_file: connector_conf = load(conf_file) if not self.connectors_configs.get(connector['type']): self.connectors_configs[connector['type']] = [] diff --git a/thingsboard_gateway/gateway/tb_logger.py b/thingsboard_gateway/gateway/tb_logger.py index 9f7e7683..2b96e1f7 100644 --- a/thingsboard_gateway/gateway/tb_logger.py +++ b/thingsboard_gateway/gateway/tb_logger.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from sys import stdout import logging import logging.handlers @@ -58,3 +59,19 @@ class TBLoggerHandler(logging.Handler): def deactivate(self): self.activated = False + + @staticmethod + def set_default_handler(): + logger_names = [ + 'service', + 'storage', + 'extension', + 'converter', + 'connector', + 'tb_connection' + ] + for logger_name in logger_names: + logger = logging.getLogger(logger_name) + handler = logging.StreamHandler(stdout) + handler.setFormatter(logging.Formatter('[STREAM ONLY] %(asctime)s - %(levelname)s - [%(filename)s] - %(module)s - %(lineno)d - %(message)s')) + logger.addHandler(handler) \ No newline at end of file