diff --git a/make_packages.sh b/make_packages.sh index 823c8816..959edb0d 100755 --- a/make_packages.sh +++ b/make_packages.sh @@ -19,13 +19,14 @@ if [ "$1" = "clean" ] || [ "$1" = "only_clean" ] ; then sudo rm -rf dist/ sudo rm -rf thingsboard-gateway.egg-info/ sudo rm -rf /etc/thingsboard-gateway/ - sudo rm -rf thingsboard-gateway-$CURRENT_VERSION.tar.gz - sudo rm -rf thingsboard-gateway-$CURRENT_VERSION.deb + sudo rm -rf thingsboard-gateway-*.tar.gz + sudo rm -rf configs.tar.gz + sudo rm -rf thingsboard-gateway-*.deb sudo rm -rf python3-thingsboard-gateway.deb sudo rm -rf python3-thingsboard-gateway.rpm - sudo rm -rf thingsboard-gateway-$CURRENT_VERSION.noarch.rpm + sudo rm -rf thingsboard-gateway-*.noarch.rpm sudo rm -rf thingsboard_gateway.egg-info - sudo rm -rf /home/zenx/rpmbuild/BUILDROOT/* + sudo rm -rf thingsboard_gateway/config/backup sudo rm -rf build/ sudo rm -rf docker/config || echo '' sudo rm -rf docker/extensions || echo '' diff --git a/setup.py b/setup.py index aeacf75b..4b123b36 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ current_directory = path.abspath(path.dirname(__file__)) with open(path.join(current_directory, 'README.md'), encoding='utf-8') as f: long_description = f.read() -VERSION = "3.4.2" +VERSION = "3.4.3.1" setup( version=VERSION, diff --git a/thingsboard-gateway.spec b/thingsboard-gateway.spec index b65e9848..fed3630c 100644 --- a/thingsboard-gateway.spec +++ b/thingsboard-gateway.spec @@ -1,6 +1,6 @@ %define name thingsboard-gateway -%define version 3.4.2 -%define unmangled_version 3.4.2 +%define version 3.4.3.1 +%define unmangled_version 3.4.3.1 %define release 1 Summary: Thingsboard Gateway for IoT devices. @@ -46,7 +46,7 @@ sudo /usr/bin/chown thingsboard_gateway:thingsboard_gateway $RPM_BUILD_ROOT/var/ # sudo find %{buildroot} -name ".pyc" -delete %post -/usr/bin/sed -i 's/\.\/logs/\/var\/log\/thingsboard-gateway/g' /etc/thingsboard-gateway/config/logs.conf >> /etc/thingsboard-gateway/config/logs.conf +/usr/bin/sed -i 's/\.\/logs/\/var\/log\/thingsboard-gateway/g' /etc/thingsboard-gateway/config/logs.json >> /etc/thingsboard-gateway/config/logs.json /usr/bin/rm -rf $RPM_BUILD_ROOT/etc/thingsboard-gateway/thingsboard-gateway /usr/bin/rm -f $RPM_BUILD_ROOT/etc/thingsboard-gateway/configs.tar.gz /usr/bin/systemctl enable thingsboard-gateway.service diff --git a/thingsboard_gateway/connectors/bacnet/bacnet_connector.py b/thingsboard_gateway/connectors/bacnet/bacnet_connector.py index e059262a..0b0c935f 100644 --- a/thingsboard_gateway/connectors/bacnet/bacnet_connector.py +++ b/thingsboard_gateway/connectors/bacnet/bacnet_connector.py @@ -280,7 +280,7 @@ class BACnetConnector(Thread, Connector): try: config_address = Address(device["address"]) device_name_tag = TBUtility.get_value(device["deviceName"], get_tag=True) - device_name = device["deviceName"].replace("${" + device_name_tag + "}", data.pop("name")) + device_name = device["deviceName"].replace("${" + device_name_tag + "}", data["name"]) device_information = { **data, **self.__get_requests_configs(device), diff --git a/thingsboard_gateway/connectors/ftp/ftp_connector.py b/thingsboard_gateway/connectors/ftp/ftp_connector.py index 016dbc1c..6ce85907 100644 --- a/thingsboard_gateway/connectors/ftp/ftp_connector.py +++ b/thingsboard_gateway/connectors/ftp/ftp_connector.py @@ -90,18 +90,19 @@ class FTPConnector(Connector, Thread): def run(self): try: - with self.__ftp() as ftp: - self.__connect(ftp) + while not self.__stopped: + with self.__ftp() as ftp: + self.__connect(ftp) - for path in self.paths: - path.find_files(ftp) - - while True: - sleep(.2) - self.__process_paths(ftp) - if self.__stopped: - break + if self._connected: + for path in self.paths: + path.find_files(ftp) + while True: + sleep(.2) + self.__process_paths(ftp) + if self.__stopped: + break except Exception as e: self.__log.exception(e) try: diff --git a/thingsboard_gateway/connectors/mqtt/mqtt_connector.py b/thingsboard_gateway/connectors/mqtt/mqtt_connector.py index 557a15b4..8b0e2fd4 100644 --- a/thingsboard_gateway/connectors/mqtt/mqtt_connector.py +++ b/thingsboard_gateway/connectors/mqtt/mqtt_connector.py @@ -13,6 +13,7 @@ # limitations under the License. import random +import socket import ssl import string from queue import Queue @@ -307,7 +308,7 @@ class MqttConnector(Connector, Thread): self._client.loop_start() if not self._connected: sleep(1) - except ConnectionRefusedError as e: + except (ConnectionRefusedError, socket.timeout) as e: self.__log.error(e) sleep(10) diff --git a/thingsboard_gateway/connectors/ocpp/ocpp_connector.py b/thingsboard_gateway/connectors/ocpp/ocpp_connector.py index 48eed137..10f6f350 100644 --- a/thingsboard_gateway/connectors/ocpp/ocpp_connector.py +++ b/thingsboard_gateway/connectors/ocpp/ocpp_connector.py @@ -105,7 +105,8 @@ class OcppConnector(Connector, Thread): self._data_convert_thread.start() self._data_send_thread.start() - self.__loop.run_until_complete(self.start_server()) + self.__loop.create_task(self.start_server()) + self.__loop.run_forever() async def start_server(self): host = self._central_system_config.get('host', '0.0.0.0') @@ -207,17 +208,18 @@ class OcppConnector(Connector, Thread): self.__stopped = True self.__connected = False - task = self.__loop.create_task(self._close_cp_connections()) - while not task.done(): - sleep(.2) + tasks = asyncio.all_tasks(self.__loop) + for task in tasks: + task.cancel() + + for socket in self._server.server.sockets: + socket._sock.close() + + self.__loop.stop() self._log.info('%s has been stopped.', self.get_name()) self._log.reset() - async def _close_cp_connections(self): - for cp in self._connected_charge_points: - await cp.close() - def get_name(self): return self.name diff --git a/thingsboard_gateway/gateway/tb_gateway_service.py b/thingsboard_gateway/gateway/tb_gateway_service.py index d6813983..bbbe6611 100644 --- a/thingsboard_gateway/gateway/tb_gateway_service.py +++ b/thingsboard_gateway/gateway/tb_gateway_service.py @@ -217,7 +217,7 @@ class TBGatewayService: self.__subscribed_to_rpc_topics = True if logging_error is not None: self.tb_client.client.send_telemetry({"ts": time() * 1000, "values": { - "LOGS": "Logging loading exception, logs.conf is wrong: %s" % (str(logging_error),)}}) + "LOGS": "Logging loading exception, logs.json is wrong: %s" % (str(logging_error),)}}) TBLoggerHandler.set_default_handler() self.__rpc_reply_sent = False self.remote_handler = TBLoggerHandler(self) @@ -225,7 +225,7 @@ class TBGatewayService: # self.main_handler.setTarget(self.remote_handler) self._default_connectors = DEFAULT_CONNECTORS self.__converted_data_queue = SimpleQueue() - self.__save_converted_data_thread = Thread(name="Save converted data", daemon=True, + self.__save_converted_data_thread = Thread(name="Storage fill thread", daemon=True, target=self.__send_to_storage) self.__save_converted_data_thread.start() self._implemented_connectors = {} @@ -252,7 +252,7 @@ class TBGatewayService: self.__rpc_processing_queue = SimpleQueue() self.__rpc_scheduled_methods_functions = { "restart": {"function": execv, "arguments": (executable, [executable.split(pathsep)[-1]] + argv)}, - "reboot": {"function": system, "arguments": ("reboot 0",)}, + "reboot": {"function": system, "arguments": ("shutdown -r now",)}, } self.__rpc_processing_thread = Thread(target=self.__send_rpc_reply_processing, daemon=True, name="RPC processing thread") @@ -520,7 +520,8 @@ class TBGatewayService: log.info("The gateway has been stopped.") self.tb_client.disconnect() self.tb_client.stop() - self.manager.shutdown() + if hasattr(self, "manager"): + self.manager.shutdown() def __init_remote_configuration(self, force=False): if (self.__config["thingsboard"].get("remoteConfiguration") or force) and self.__remote_configurator is None: @@ -731,7 +732,7 @@ class TBGatewayService: connector.get('class'))) if connector_class is None: - log.warning("Connector implementation not found for %s", connector["name"]) + log.warning("Connector implementation not found for %s", connector['name']) else: self._implemented_connectors[connector['type']] = connector_class elif connector['type'] == "grpc": @@ -758,10 +759,8 @@ class TBGatewayService: if connector['type'] != 'grpc' and isinstance(connector_conf, dict): connector_conf["name"] = connector['name'] self.connectors_configs[connector['type']].append({"name": connector['name'], - "config": {connector[ - 'configuration']: connector_conf} if - connector[ - 'type'] != 'grpc' else connector_conf, + "config": {connector['configuration']: connector_conf} if + connector['type'] != 'grpc' else connector_conf, "config_updated": stat(config_file_path), "config_file_path": config_file_path, "grpc_key": connector_persistent_key}) @@ -786,7 +785,7 @@ class TBGatewayService: connector = None connector_name = None try: - if connector_config["config"][config] is not None: + if connector_config["config"][config] is not None and len(connector_config["config"][config].keys()) > 2: connector_name = connector_config["name"] if not self.available_connectors.get(connector_name): @@ -800,7 +799,7 @@ class TBGatewayService: else: break else: - log.info("Config not found for %s", connector_type) + log.warning("Config not found or empty for %s", connector_type) except Exception as e: log.exception(e, attr_name=connector_name) if connector is not None: diff --git a/thingsboard_gateway/tb_utility/tb_gateway_remote_configurator.py b/thingsboard_gateway/tb_utility/tb_gateway_remote_configurator.py index 14015b6c..f628c8f5 100644 --- a/thingsboard_gateway/tb_utility/tb_gateway_remote_configurator.py +++ b/thingsboard_gateway/tb_utility/tb_gateway_remote_configurator.py @@ -23,7 +23,6 @@ from packaging import version from thingsboard_gateway.gateway.tb_client import TBClient from thingsboard_gateway.tb_utility.tb_handler import TBLoggerHandler -from thingsboard_gateway.connectors.converter import Converter LOG = getLogger("service") @@ -100,7 +99,9 @@ class RemoteConfigurator: def callback(key, err): if err is not None: LOG.exception(err) + if key is None: self._remote_gateway_version = '0.0' + return try: self._remote_gateway_version = key['client']['Version'] @@ -125,7 +126,10 @@ class RemoteConfigurator: try_count += 1 sleep(1) - need_update_configs = version.parse(self._gateway.version.get('current_version', '0.0')) > version.parse( + if self._remote_gateway_version is None: + self._remote_gateway_version = "0.0" + + need_update_configs = self._remote_gateway_version == "0.0" or version.parse(self._gateway.version.get('current_version', '0.0')) > version.parse( str(self._remote_gateway_version)) if need_update_configs: diff --git a/thingsboard_gateway/tb_utility/tb_loader.py b/thingsboard_gateway/tb_utility/tb_loader.py index 7ff1f1b1..05656462 100644 --- a/thingsboard_gateway/tb_utility/tb_loader.py +++ b/thingsboard_gateway/tb_utility/tb_loader.py @@ -69,7 +69,7 @@ class TBModuleLoader: TBModuleLoader.LOADED_CONNECTORS[buffered_module_name] = extension_class[1] return extension_class[1] except ImportError as e: - log.exception(e) + log.error(e.msg) continue except Exception as e: log.exception(e) diff --git a/thingsboard_gateway/tb_utility/tb_updater.py b/thingsboard_gateway/tb_utility/tb_updater.py index bb228021..10653fa9 100644 --- a/thingsboard_gateway/tb_utility/tb_updater.py +++ b/thingsboard_gateway/tb_utility/tb_updater.py @@ -38,7 +38,7 @@ class TBUpdater(Thread): self.__version = {"current_version": get_distribution('thingsboard_gateway').version, "latest_version": get_distribution('thingsboard_gateway').version} except DistributionNotFound: - self.__version = {"current_version": "0", "latest_version": "0"} + self.__version = {"current_version": "0.0", "latest_version": "0.0"} self.__instance_id = str(uuid1()) self.__platform = system() diff --git a/thingsboard_gateway/tb_utility/tb_utility.py b/thingsboard_gateway/tb_utility/tb_utility.py index a2630452..927d9d9f 100644 --- a/thingsboard_gateway/tb_utility/tb_utility.py +++ b/thingsboard_gateway/tb_utility/tb_utility.py @@ -101,15 +101,14 @@ class TBUtility: try: if isinstance(body, dict) and target_str.split()[0] in body: if value_type.lower() == "string": - full_value = str(expression[0: max(p1 - 2, 0)]) + str(body[target_str.split()[0]]) + str(expression[ - p2 + 1:len( - expression)]) + full_value = str(expression[0: max(p1 - 2, 0)]) + str(body[target_str.split()[0]]) + str(expression[p2 + 1:len(expression)]) else: full_value = body.get(target_str.split()[0]) elif isinstance(body, (dict, list)): try: - # Wrap in quotes to support key name with spaces - jsonpath_expression = parse('"' + target_str + '"') + if " " in target_str: + target_str = '.'.join('"' + section_key + '"' if " " in section_key else section_key for section_key in target_str.split('.')) + jsonpath_expression = parse(target_str) jsonpath_match = jsonpath_expression.find(body) if jsonpath_match: full_value = jsonpath_match[0].value @@ -141,6 +140,8 @@ class TBUtility: def install_package(package, version="upgrade", force_install=False): from sys import executable from subprocess import check_call, CalledProcessError + import site + from importlib import reload result = False if force_install: @@ -168,6 +169,13 @@ class TBUtility: [executable, "-m", "pip", "install", package + installation_sign + version, "--user"]) except CalledProcessError: result = check_call([executable, "-m", "pip", "install", package + installation_sign + version]) + + # Because `pip` is running in a subprocess the newly installed modules and libraries are + # not immediately available to the current runtime. + # Refreshing sys.path fixes this. See: + # https://stackoverflow.com/questions/4271494/what-sets-up-sys-path-with-python-and-when + reload(site) + return result @staticmethod @@ -236,4 +244,4 @@ class TBUtility: elif 'bool' in new_type: return bool(evaluated_data) else: - return str(evaluated_data) \ No newline at end of file + return str(evaluated_data)