From e86a07f7f9d8b6faebb457fcc55c6f963289d1d1 Mon Sep 17 00:00:00 2001 From: wangcaigg Date: Sun, 26 Nov 2023 19:31:11 +0800 Subject: [PATCH 1/7] fixbug:the rpc request in progress got replaced by the new rpc request --- thingsboard_gateway/gateway/tb_gateway_service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/thingsboard_gateway/gateway/tb_gateway_service.py b/thingsboard_gateway/gateway/tb_gateway_service.py index e3686f74..d6813983 100644 --- a/thingsboard_gateway/gateway/tb_gateway_service.py +++ b/thingsboard_gateway/gateway/tb_gateway_service.py @@ -453,6 +453,7 @@ class TBGatewayService: new_rpc_request_in_progress = {key: value for key, value in self.__rpc_requests_in_progress.items() if value != 'del'} if not self.__rpc_register_queue.empty(): + new_rpc_request_in_progress = self.__rpc_requests_in_progress rpc_request_from_queue = self.__rpc_register_queue.get(False) topic = rpc_request_from_queue["topic"] data = rpc_request_from_queue["data"] From b07e2b59a5cdd184066cf9d2cc89c1c4eaa8e3a2 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Fri, 26 Jan 2024 13:52:32 +0200 Subject: [PATCH 2/7] Updated connector saving due to issue #1268 --- .../gateway/tb_gateway_service.py | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/thingsboard_gateway/gateway/tb_gateway_service.py b/thingsboard_gateway/gateway/tb_gateway_service.py index 89fb2d14..0eccaf4f 100644 --- a/thingsboard_gateway/gateway/tb_gateway_service.py +++ b/thingsboard_gateway/gateway/tb_gateway_service.py @@ -720,31 +720,35 @@ class TBGatewayService: for connector in configuration: try: connector_persistent_key = None - if connector['type'] == "grpc" and self.__grpc_manager is None: + connector_type = connector["type"].lower() if connector.get("type") is not None else None + if connector_type is None: + log.error("Connector type is not defined!") + continue + if connector_type == "grpc" and self.__grpc_manager is None: log.error("Cannot load connector with name: %s and type grpc. GRPC server is disabled!", connector['name']) continue - if connector['type'] != "grpc": + if connector_type != "grpc": connector_class = None if connector.get('useGRPC', False): - module_name = f'Grpc{self._default_connectors.get(connector["type"], connector.get("class"))}' - connector_class = TBModuleLoader.import_module(connector['type'], module_name) + module_name = f'Grpc{self._default_connectors.get(connector_type, connector.get("class"))}' + connector_class = TBModuleLoader.import_module(connector_type, module_name) if self.__grpc_manager and self.__grpc_manager.is_alive() and connector_class: connector_persistent_key = self._generate_persistent_key(connector, connectors_persistent_keys) else: - connector_class = TBModuleLoader.import_module(connector['type'], + connector_class = TBModuleLoader.import_module(connector_type, self._default_connectors.get( - connector['type'], + connector_type, connector.get('class'))) if connector_class is None: log.warning("Connector implementation not found for %s", connector['name']) else: - self._implemented_connectors[connector['type']] = connector_class - elif connector['type'] == "grpc": + self._implemented_connectors[connector_type] = connector_class + elif connector_type == "grpc": if connector.get('key') == "auto": self._generate_persistent_key(connector, connectors_persistent_keys) else: @@ -776,14 +780,14 @@ class TBGatewayService: if not (start_find > -1 and end_find > -1): connector_conf = "{id_var_start}" + str(connector_id) + "{id_var_end}" + connector_conf - if not self.connectors_configs.get(connector['type']): - self.connectors_configs[connector['type']] = [] - if connector['type'] != 'grpc' and isinstance(connector_conf, dict): + if not self.connectors_configs.get(connector_type): + self.connectors_configs[connector_type] = [] + if connector_type != 'grpc' and isinstance(connector_conf, dict): connector_conf["name"] = connector['name'] - self.connectors_configs[connector['type']].append({"name": connector['name'], + self.connectors_configs[connector_type].append({"name": connector['name'], "id": connector_id, "config": {connector['configuration']: connector_conf} if - connector['type'] != 'grpc' else connector_conf, + connector_type != 'grpc' else connector_conf, "config_updated": stat(config_file_path), "config_file_path": config_file_path, "grpc_key": connector_persistent_key}) @@ -802,9 +806,10 @@ class TBGatewayService: def __connect_with_connectors(self): global log for connector_type in self.connectors_configs: + connector_type = connector_type.lower() for connector_config in self.connectors_configs[connector_type]: - if self._implemented_connectors.get(connector_type.lower()) is not None: - if connector_type.lower() != 'grpc' and 'Grpc' not in self._implemented_connectors[connector_type.lower()].__name__: + if self._implemented_connectors.get(connector_type) is not None: + if connector_type != 'grpc' and 'Grpc' not in self._implemented_connectors[connector_type].__name__: for config in connector_config["config"]: connector = None connector_name = None @@ -839,7 +844,7 @@ class TBGatewayService: connector.close() else: self.__grpc_connectors.update({connector_config['grpc_key']: connector_config}) - if connector_type.lower() != 'grpc': + if connector_type != 'grpc': connector_dir_abs = "/".join(self._config_dir.split("/")[:-2]) connector_file_name = f'{connector_type}_connector.py' connector_abs_path = f'{connector_dir_abs}/grpc_connectors/{connector_type}/{connector_file_name}' From 29658ab1da97ab4951b20b19cb753b530b38757e Mon Sep 17 00:00:00 2001 From: imbeacon Date: Fri, 26 Jan 2024 14:09:37 +0200 Subject: [PATCH 3/7] Fix for issue described in #1195 --- thingsboard_gateway/connectors/mqtt/mqtt_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thingsboard_gateway/connectors/mqtt/mqtt_connector.py b/thingsboard_gateway/connectors/mqtt/mqtt_connector.py index 89bd2ea1..aad54b17 100644 --- a/thingsboard_gateway/connectors/mqtt/mqtt_connector.py +++ b/thingsboard_gateway/connectors/mqtt/mqtt_connector.py @@ -312,7 +312,7 @@ class MqttConnector(Connector, Thread): self._client.loop_start() if not self._connected: sleep(1) - except (ConnectionRefusedError, socket.timeout) as e: + except (ConnectionRefusedError, ConnectionResetError, socket.timeout) as e: self.__log.error(e) sleep(10) From 308ef9a4190ee1053c2d6371c67e0390111a614a Mon Sep 17 00:00:00 2001 From: imbeacon Date: Tue, 30 Jan 2024 12:26:36 +0200 Subject: [PATCH 4/7] Fix for saved device, with not found connector by id --- thingsboard_gateway/gateway/tb_gateway_service.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/thingsboard_gateway/gateway/tb_gateway_service.py b/thingsboard_gateway/gateway/tb_gateway_service.py index 0eccaf4f..8a8280e6 100644 --- a/thingsboard_gateway/gateway/tb_gateway_service.py +++ b/thingsboard_gateway/gateway/tb_gateway_service.py @@ -1471,8 +1471,19 @@ class TBGatewayService: new_device_name = loaded_connected_devices[device_name][2] self.__renamed_devices[device_name] = new_device_name elif isinstance(loaded_connected_devices[device_name], dict): + connector = None + if not self.available_connectors_by_id.get(loaded_connected_devices[device_name][CONNECTOR_ID_PARAMETER]): + log.warning("Connector with id %s not found, trying to use connector by name!", loaded_connected_devices[device_name][CONNECTOR_ID_PARAMETER]) + connector = self.available_connectors_by_name.get(loaded_connected_devices[device_name][CONNECTOR_NAME_PARAMETER]) + else: + connector = self.available_connectors_by_id.get(loaded_connected_devices[device_name][CONNECTOR_ID_PARAMETER]) + if connector is None: + log.warning("Connector with name %s not found! probably it is disabled, device %s will be " + "removed from the saved devices", + loaded_connected_devices[device_name][CONNECTOR_NAME_PARAMETER], device_name) + continue device_data_to_save = { - CONNECTOR_PARAMETER: self.available_connectors_by_id[loaded_connected_devices[device_name][CONNECTOR_ID_PARAMETER]], + CONNECTOR_PARAMETER: connector, DEVICE_TYPE_PARAMETER: loaded_connected_devices[device_name][DEVICE_TYPE_PARAMETER]} if loaded_connected_devices[device_name].get(RENAMING_PARAMETER) is not None: new_device_name = loaded_connected_devices[device_name][RENAMING_PARAMETER] From c57624d5669139b651cd69a6656be7b20ee8f86e Mon Sep 17 00:00:00 2001 From: imbeacon Date: Tue, 30 Jan 2024 12:27:33 +0200 Subject: [PATCH 5/7] Fix for #1267, fixed starting already started thread --- thingsboard_gateway/tb_utility/tb_handler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/thingsboard_gateway/tb_utility/tb_handler.py b/thingsboard_gateway/tb_utility/tb_handler.py index 467bb1ab..2b66846d 100644 --- a/thingsboard_gateway/tb_utility/tb_handler.py +++ b/thingsboard_gateway/tb_utility/tb_handler.py @@ -40,7 +40,7 @@ class TBLoggerHandler(logging.Handler): self._max_message_count_batch = 20 self._logs_queue = Queue(1000) - # start() method calls in activate() method + self._send_logs_thread = threading.Thread(target=self._send_logs, name='Logs Sending Thread', daemon=True) self.setFormatter(logging.Formatter('%(asctime)s - |%(levelname)s| - [%(filename)s] - %(module)s - %(lineno)d - %(message)s')) @@ -105,7 +105,8 @@ class TBLoggerHandler(logging.Handler): log = TbLogger('service') log.exception(e) self.activated = True - self._send_logs_thread.start() + if not self._send_logs_thread.is_alive(): + self._send_logs_thread.start() def handle(self, record): if self.activated and not self.__gateway.stopped: From 671515682a17d788a9dd820b43fb97320a7ba6ec Mon Sep 17 00:00:00 2001 From: imbeacon Date: Tue, 30 Jan 2024 14:25:25 +0200 Subject: [PATCH 6/7] Added additional env variables to avoid matching with default for systems, issue #1255 --- thingsboard_gateway/gateway/tb_gateway_service.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/thingsboard_gateway/gateway/tb_gateway_service.py b/thingsboard_gateway/gateway/tb_gateway_service.py index 8a8280e6..2bb65aa6 100644 --- a/thingsboard_gateway/gateway/tb_gateway_service.py +++ b/thingsboard_gateway/gateway/tb_gateway_service.py @@ -114,6 +114,18 @@ def get_env_variables(): 'password': environ.get('password') } + env_variables = { + 'host': environ.get('TB_GW_HOST'), + 'port': int(environ.get('TB_GW_PORT')) if environ.get('TB_GW_PORT') else None, + 'accessToken': environ.get('TB_GW_ACCESS_TOKEN'), + 'caCert': environ.get('TB_GW_CA_CERT'), + 'privateKey': environ.get('TB_GW_PRIVATE_KEY'), + 'cert': environ.get('TB_GW_CERT'), + 'clientId': environ.get('TB_GW_CLIENT_ID'), + 'username': environ.get('TB_GW_USERNAME'), + 'password': environ.get('TB_GW_PASSWORD') + } + converted_env_variables = {} for (key, value) in env_variables.items(): From fbdfe7ec98f7b270765487ace2defe7ee60511ba Mon Sep 17 00:00:00 2001 From: imbeacon Date: Tue, 30 Jan 2024 14:29:03 +0200 Subject: [PATCH 7/7] Updated env assignment --- .../gateway/tb_gateway_service.py | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/thingsboard_gateway/gateway/tb_gateway_service.py b/thingsboard_gateway/gateway/tb_gateway_service.py index 2bb65aa6..550819bf 100644 --- a/thingsboard_gateway/gateway/tb_gateway_service.py +++ b/thingsboard_gateway/gateway/tb_gateway_service.py @@ -114,17 +114,24 @@ def get_env_variables(): 'password': environ.get('password') } - env_variables = { - 'host': environ.get('TB_GW_HOST'), - 'port': int(environ.get('TB_GW_PORT')) if environ.get('TB_GW_PORT') else None, - 'accessToken': environ.get('TB_GW_ACCESS_TOKEN'), - 'caCert': environ.get('TB_GW_CA_CERT'), - 'privateKey': environ.get('TB_GW_PRIVATE_KEY'), - 'cert': environ.get('TB_GW_CERT'), - 'clientId': environ.get('TB_GW_CLIENT_ID'), - 'username': environ.get('TB_GW_USERNAME'), - 'password': environ.get('TB_GW_PASSWORD') - } + if environ.get('TB_GW_HOST'): + env_variables['host'] = environ.get('TB_GW_HOST') + if environ.get('TB_GW_PORT'): + env_variables['port'] = int(environ.get('TB_GW_PORT')) + if environ.get('TB_GW_ACCESS_TOKEN'): + env_variables['accessToken'] = environ.get('TB_GW_ACCESS_TOKEN') + if environ.get('TB_GW_CA_CERT'): + env_variables['caCert'] = environ.get('TB_GW_CA_CERT') + if environ.get('TB_GW_PRIVATE_KEY'): + env_variables['privateKey'] = environ.get('TB_GW_PRIVATE_KEY') + if environ.get('TB_GW_CERT'): + env_variables['cert'] = environ.get('TB_GW_CERT') + if environ.get('TB_GW_CLIENT_ID'): + env_variables['clientId'] = environ.get('TB_GW_CLIENT_ID') + if environ.get('TB_GW_USERNAME'): + env_variables['username'] = environ.get('TB_GW_USERNAME') + if environ.get('TB_GW_PASSWORD'): + env_variables['password'] = environ.get('TB_GW_PASSWORD') converted_env_variables = {}