From fbef042ecee624daf3865aac62d5d28103f562ae Mon Sep 17 00:00:00 2001 From: zbeacon Date: Wed, 29 Apr 2020 09:39:50 +0300 Subject: [PATCH 1/7] Modbus connector improvements --- .../connectors/modbus/modbus_connector.py | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/thingsboard_gateway/connectors/modbus/modbus_connector.py b/thingsboard_gateway/connectors/modbus/modbus_connector.py index dad87a1a..3d082d73 100644 --- a/thingsboard_gateway/connectors/modbus/modbus_connector.py +++ b/thingsboard_gateway/connectors/modbus/modbus_connector.py @@ -185,7 +185,20 @@ class ModbusConnector(Connector, threading.Thread): log.exception(e) def on_attributes_update(self, content): - pass + try: + for attribute_updates_command_config in self.__devices[content["device"]]["config"]["attributeUpdates"]: + for attribute_updated in content["data"]: + if attribute_updates_command_config["tag"] == attribute_updated: + to_process = { + "device": content["device"], + "data": { + "method": attribute_updated, + "params": content["data"][attribute_updated] + } + } + self.__process_rpc_request(to_process, attribute_updates_command_config) + except Exception as e: + log.exception(e) def __configure_master(self): host = self.__config.get("host", "localhost") @@ -293,12 +306,13 @@ class ModbusConnector(Connector, threading.Thread): WriteSingleRegisterResponse)): log.debug("Write %r", str(response)) response = {"success": True} - if isinstance(response, Exception): - self.__gateway.send_rpc_reply(content["device"], - content["data"]["id"], - {content["data"]["method"]: str(response)}) - else: - self.__gateway.send_rpc_reply(content["device"], - content["data"]["id"], - response) + if content.get("id"): + if isinstance(response, Exception): + self.__gateway.send_rpc_reply(content["device"], + content["data"]["id"], + {content["data"]["method"]: str(response)}) + else: + self.__gateway.send_rpc_reply(content["device"], + content["data"]["id"], + response) log.debug("%r", response) From a2f80b2db3fe37c4691d7f0de61f41d3b5fc9a9c Mon Sep 17 00:00:00 2001 From: zbeacon Date: Wed, 29 Apr 2020 09:40:18 +0300 Subject: [PATCH 2/7] MQTT Connector improvements on connection refused --- thingsboard_gateway/connectors/mqtt/mqtt_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thingsboard_gateway/connectors/mqtt/mqtt_connector.py b/thingsboard_gateway/connectors/mqtt/mqtt_connector.py index 753793d5..99729ad5 100644 --- a/thingsboard_gateway/connectors/mqtt/mqtt_connector.py +++ b/thingsboard_gateway/connectors/mqtt/mqtt_connector.py @@ -175,7 +175,7 @@ class MqttConnector(Connector, Thread): self._client.loop_start() if not self._connected: time.sleep(1) - except Exception as e: + except ConnectionRefusedError as e: self.__log.exception(e) time.sleep(10) From 4b17837f84bccaceaf9fe366705ee6548bc7abba Mon Sep 17 00:00:00 2001 From: zbeacon Date: Wed, 29 Apr 2020 10:02:14 +0300 Subject: [PATCH 3/7] Modbus Connector type "bit" will be deprecated, please use "bits" --- thingsboard_gateway/config/modbus.json | 4 ++-- .../modbus/bytes_modbus_downlink_converter.py | 9 +++++---- .../connectors/modbus/bytes_modbus_uplink_converter.py | 10 +++------- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/thingsboard_gateway/config/modbus.json b/thingsboard_gateway/config/modbus.json index 83976284..e8ed9745 100644 --- a/thingsboard_gateway/config/modbus.json +++ b/thingsboard_gateway/config/modbus.json @@ -136,14 +136,14 @@ "rpc": [ { "tag": "setValue", - "type": "bit", + "type": "bits", "functionCode": 5, "objectsCount": 1, "address": 31 }, { "tag": "getValue", - "type": "bit", + "type": "bits", "functionCode": 1, "objectsCount": 1, "address": 31 diff --git a/thingsboard_gateway/connectors/modbus/bytes_modbus_downlink_converter.py b/thingsboard_gateway/connectors/modbus/bytes_modbus_downlink_converter.py index 7d00dbc7..4cab2304 100644 --- a/thingsboard_gateway/connectors/modbus/bytes_modbus_downlink_converter.py +++ b/thingsboard_gateway/connectors/modbus/bytes_modbus_downlink_converter.py @@ -62,14 +62,15 @@ class BytesModbusDownlinkConverter(ModbusConverter): lower_type = str(variable_size) + "float" assert builder_functions.get(lower_type) is not None builder_functions[lower_type](value) - elif lower_type in ["coil", "bits"]: + elif lower_type in ["coil", "bits", "coils", "bit"]: assert builder_functions.get("bits") is not None - builder_functions["bits"](value) + if variable_size/8 > 1.0: + builder_functions["bits"](value) + else: + return bytes(bool(int(value))) elif lower_type in ["string"]: assert builder_functions.get("string") is not None builder_functions[lower_type](value) - elif lower_type in ["bit"]: - return bytes(bool(int(value))) elif lower_type in builder_functions: builder_functions[lower_type](value) else: diff --git a/thingsboard_gateway/connectors/modbus/bytes_modbus_uplink_converter.py b/thingsboard_gateway/connectors/modbus/bytes_modbus_uplink_converter.py index a530549c..e7e6a11d 100644 --- a/thingsboard_gateway/connectors/modbus/bytes_modbus_uplink_converter.py +++ b/thingsboard_gateway/connectors/modbus/bytes_modbus_uplink_converter.py @@ -47,9 +47,9 @@ class BytesModbusUplinkConverter(ModbusConverter): result = response.bits result = result if byte_order.upper() == 'LITTLE' else result[::-1] log.debug(result) - if configuration["type"].lower() == "bit": + if configuration["type"].lower() == "bits": decoded_data = result[:configuration.get("objectsCount", configuration.get("registersCount", configuration.get("registerCount", 1)))] - if len(decoded_data) == 1: + if len(decoded_data) == 1 and isinstance(decoded_data, list): decoded_data = decoded_data[0] else: decoded_data = result[0] @@ -127,10 +127,6 @@ class BytesModbusUplinkConverter(ModbusConverter): assert decoder_functions.get(type_) is not None decoded = decoder_functions[type_]() - elif lower_type == 'bit': - bit_address = configuration["bit"] - decoded = decoder_functions[type_]()[bit_address] - else: log.error("Unknown type: %s", type_) @@ -141,7 +137,7 @@ class BytesModbusUplinkConverter(ModbusConverter): elif isinstance(decoded, bytes) and lower_type == "bytes": result_data = decoded elif isinstance(decoded, list): - result_data = str(decoded) + result_data = decoded elif isinstance(decoded, float): result_data = decoded elif decoded is not None: From 0273fa1e6a32be2e66f830d83521f92d8d03d245 Mon Sep 17 00:00:00 2001 From: zbeacon Date: Fri, 1 May 2020 12:57:49 +0300 Subject: [PATCH 4/7] #284 Fix for attribute error --- thingsboard_gateway/tb_utility/tb_utility.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/thingsboard_gateway/tb_utility/tb_utility.py b/thingsboard_gateway/tb_utility/tb_utility.py index fa158b07..740977e2 100644 --- a/thingsboard_gateway/tb_utility/tb_utility.py +++ b/thingsboard_gateway/tb_utility/tb_utility.py @@ -47,7 +47,10 @@ class TBUtility: error = 'No telemetry and attributes in data: ' if error is not None: json_data = dumps(data) - log.error(error+json_data.decode("UTF-8")) + if isinstance(json_data, bytes): + log.error(error+json_data.decode("UTF-8")) + else: + log.error(error + json_data) return False return True From dcdda512d254eb0e541327aee341e1358e791e5f Mon Sep 17 00:00:00 2001 From: zbeacon Date: Mon, 4 May 2020 08:35:42 +0300 Subject: [PATCH 5/7] ByteOrder fix and RPC reply fix --- .../connectors/modbus/bytes_modbus_uplink_converter.py | 4 ++-- thingsboard_gateway/connectors/modbus/modbus_connector.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/thingsboard_gateway/connectors/modbus/bytes_modbus_uplink_converter.py b/thingsboard_gateway/connectors/modbus/bytes_modbus_uplink_converter.py index e7e6a11d..95dd5a5e 100644 --- a/thingsboard_gateway/connectors/modbus/bytes_modbus_uplink_converter.py +++ b/thingsboard_gateway/connectors/modbus/bytes_modbus_uplink_converter.py @@ -36,8 +36,8 @@ class BytesModbusUplinkConverter(ModbusConverter): try: configuration = data[config_data][tag]["data_sent"] response = data[config_data][tag]["input_data"] - if config.get("byteOrder"): - byte_order = config["byteOrder"] + if configuration.get("byteOrder") is not None: + byte_order = configuration["byteOrder"] else: byte_order = configuration.get("byteOrder", "LITTLE") endian_order = Endian.Little if byte_order.upper() == "LITTLE" else Endian.Big diff --git a/thingsboard_gateway/connectors/modbus/modbus_connector.py b/thingsboard_gateway/connectors/modbus/modbus_connector.py index 3d082d73..eb153387 100644 --- a/thingsboard_gateway/connectors/modbus/modbus_connector.py +++ b/thingsboard_gateway/connectors/modbus/modbus_connector.py @@ -259,6 +259,7 @@ class ModbusConnector(Connector, threading.Thread): def server_side_rpc_handler(self, content): try: if content.get("device") is not None: + log.debug("Modbus connector received rpc request for %s with content: %s", content["device"], content) if isinstance(self.__devices[content["device"]]["config"]["rpc"], dict): rpc_command_config = self.__devices[content["device"]]["config"]["rpc"].get(content["data"]["method"]) @@ -268,6 +269,7 @@ class ModbusConnector(Connector, threading.Thread): for rpc_command_config in self.__devices[content["device"]]["config"]["rpc"]: if rpc_command_config["tag"] == content["data"]["method"]: self.__process_rpc_request(content, rpc_command_config) + break else: log.error("Received rpc request, but method %s not found in config for %s.", content["data"].get("method"), @@ -306,7 +308,7 @@ class ModbusConnector(Connector, threading.Thread): WriteSingleRegisterResponse)): log.debug("Write %r", str(response)) response = {"success": True} - if content.get("id"): + if content.get("id") or (content.get("data") is not None and content["data"].get("id")): if isinstance(response, Exception): self.__gateway.send_rpc_reply(content["device"], content["data"]["id"], From a5425a72bc36b10f8da9e2c172ecf5e0b2b4c9ac Mon Sep 17 00:00:00 2001 From: zbeacon Date: Mon, 4 May 2020 13:32:33 +0300 Subject: [PATCH 6/7] Added required library installation into connectors instead of installing during general installation. To decrease traffic and module size. --- setup.py | 6 +----- .../connectors/bacnet/bacnet_connector.py | 14 +++++++++++--- .../connectors/can/can_connector.py | 12 +++++++++--- .../connectors/modbus/modbus_connector.py | 11 +++++++++-- .../connectors/opcua/opcua_connector.py | 7 ++++++- .../connectors/request/request_connector.py | 18 ++++++++++++------ .../gateway/tb_gateway_service.py | 1 + thingsboard_gateway/tb_utility/tb_utility.py | 9 +++++++++ 8 files changed, 58 insertions(+), 20 deletions(-) diff --git a/setup.py b/setup.py index 92d2be48..452df1ca 100644 --- a/setup.py +++ b/setup.py @@ -51,17 +51,13 @@ setup( 'pip', 'jsonschema==3.1.1', 'lxml', - 'opcua', 'paho-mqtt', - 'pymodbus>=2.3.0', 'pyserial', 'pytz', 'PyYAML', 'simplejson', 'pyrsistent', - 'requests', - 'python-can', - 'bacpypes>=0.18.0' + 'requests' ], download_url='https://github.com/thingsboard/thingsboard-gateway/archive/%s.tar.gz' % VERSION, entry_points={ diff --git a/thingsboard_gateway/connectors/bacnet/bacnet_connector.py b/thingsboard_gateway/connectors/bacnet/bacnet_connector.py index c02abbf6..329da38d 100644 --- a/thingsboard_gateway/connectors/bacnet/bacnet_connector.py +++ b/thingsboard_gateway/connectors/bacnet/bacnet_connector.py @@ -12,16 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -from copy import deepcopy from random import choice from threading import Thread from time import time, sleep from string import ascii_lowercase -from bacpypes.core import run, stop + +from thingsboard_gateway.tb_utility.tb_utility import TBUtility + +try: + from bacpypes.core import run, stop +except ImportError: + print("BACnet library not found - installing...") + TBUtility.install_package("bacpypes", ">=0.18.0") + from bacpypes.core import run, stop + from bacpypes.pdu import Address, GlobalBroadcast, LocalBroadcast, LocalStation, RemoteStation + from thingsboard_gateway.connectors.connector import Connector, log from thingsboard_gateway.connectors.bacnet.bacnet_utilities.tb_gateway_bacnet_application import TBBACnetApplication -from thingsboard_gateway.tb_utility.tb_utility import TBUtility class BACnetConnector(Thread, Connector): diff --git a/thingsboard_gateway/connectors/can/can_connector.py b/thingsboard_gateway/connectors/can/can_connector.py index 1042bc41..1c17cafc 100644 --- a/thingsboard_gateway/connectors/can/can_connector.py +++ b/thingsboard_gateway/connectors/can/can_connector.py @@ -15,16 +15,22 @@ import re import sched import time +from threading import Thread from copy import copy from random import choice from string import ascii_lowercase -from can import Notifier, BufferedReader, Message, CanError, ThreadSafeBus -from threading import Thread + +from thingsboard_gateway.tb_utility.tb_utility import TBUtility +try: + from can import Notifier, BufferedReader, Message, CanError, ThreadSafeBus +except ImportError: + print("CAN library not found - installing...") + TBUtility.install_package("python-can") + from can import Notifier, BufferedReader, Message, CanError, ThreadSafeBus from thingsboard_gateway.connectors.can.bytes_can_downlink_converter import BytesCanDownlinkConverter from thingsboard_gateway.connectors.can.bytes_can_uplink_converter import BytesCanUplinkConverter from thingsboard_gateway.connectors.connector import Connector, log -from thingsboard_gateway.tb_utility.tb_utility import TBUtility class CanConnector(Connector, Thread): diff --git a/thingsboard_gateway/connectors/modbus/modbus_connector.py b/thingsboard_gateway/connectors/modbus/modbus_connector.py index eb153387..55ca276e 100644 --- a/thingsboard_gateway/connectors/modbus/modbus_connector.py +++ b/thingsboard_gateway/connectors/modbus/modbus_connector.py @@ -17,7 +17,15 @@ import threading from random import choice from string import ascii_lowercase -from pymodbus.constants import Defaults +from thingsboard_gateway.tb_utility.tb_utility import TBUtility +# Try import Pymodbus library or install it and import +try: + from pymodbus.constants import Defaults +except ImportError: + print("Modbus library not found - installing...") + TBUtility.install_package("pymodbus", ">=2.3.0") + from pymodbus.constants import Defaults + from pymodbus.client.sync import ModbusTcpClient, ModbusUdpClient, ModbusSerialClient, ModbusRtuFramer, ModbusSocketFramer from pymodbus.bit_write_message import WriteSingleCoilResponse, WriteMultipleCoilsResponse from pymodbus.register_write_message import WriteMultipleRegistersResponse, \ @@ -26,7 +34,6 @@ from pymodbus.register_read_message import ReadRegistersResponseBase from pymodbus.bit_read_message import ReadBitsResponseBase from pymodbus.exceptions import ConnectionException -from thingsboard_gateway.tb_utility.tb_utility import TBUtility from thingsboard_gateway.connectors.connector import Connector, log from thingsboard_gateway.connectors.modbus.bytes_modbus_uplink_converter import BytesModbusUplinkConverter from thingsboard_gateway.connectors.modbus.bytes_modbus_downlink_converter import BytesModbusDownlinkConverter diff --git a/thingsboard_gateway/connectors/opcua/opcua_connector.py b/thingsboard_gateway/connectors/opcua/opcua_connector.py index 6ffd8e61..ccd35487 100644 --- a/thingsboard_gateway/connectors/opcua/opcua_connector.py +++ b/thingsboard_gateway/connectors/opcua/opcua_connector.py @@ -20,9 +20,14 @@ from random import choice from threading import Thread from string import ascii_lowercase import regex -from opcua import Client, ua from simplejson import dumps from thingsboard_gateway.tb_utility.tb_utility import TBUtility +try: + from opcua import Client, ua +except ImportError: + print("OPC-UA library not found") + TBUtility.install_package("opcua") + from opcua import Client, ua from thingsboard_gateway.connectors.connector import Connector, log from thingsboard_gateway.connectors.opcua.opcua_uplink_converter import OpcUaUplinkConverter diff --git a/thingsboard_gateway/connectors/request/request_connector.py b/thingsboard_gateway/connectors/request/request_connector.py index 78698344..c91b2620 100644 --- a/thingsboard_gateway/connectors/request/request_connector.py +++ b/thingsboard_gateway/connectors/request/request_connector.py @@ -19,12 +19,18 @@ from string import ascii_lowercase from time import sleep, time from re import fullmatch -import requests -from requests import Timeout +# import requests +from thingsboard_gateway.tb_utility.tb_utility import TBUtility +try: + from requests import Timeout, request +except ImportError: + print("Requests library not found - installing...") + TBUtility.install_package("requests") + from requests import Timeout, request + from requests.auth import HTTPBasicAuth from requests.exceptions import RequestException -from thingsboard_gateway.tb_utility.tb_utility import TBUtility from thingsboard_gateway.connectors.connector import Connector, log from thingsboard_gateway.connectors.request.json_request_uplink_converter import JsonRequestUplinkConverter from thingsboard_gateway.connectors.request.json_request_downlink_converter import JsonRequestDownlinkConverter @@ -80,7 +86,7 @@ class RequestConnector(Connector, Thread): response_queue = Queue(1) request_dict = {"config": {**attribute_request, **converted_data}, - "request": requests.request} + "request": request} attribute_update_request_thread = Thread(target=self.__send_request, args=(request_dict, response_queue, log), daemon=True, @@ -102,7 +108,7 @@ class RequestConnector(Connector, Thread): response_queue = Queue(1) request_dict = {"config": {**rpc_request, **converted_data}, - "request": requests.request} + "request": request} request_dict["config"].get("uplink_converter") rpc_request_thread = Thread(target=self.__send_request, args=(request_dict, response_queue, log), @@ -138,7 +144,7 @@ class RequestConnector(Connector, Thread): self.__requests_in_progress.append({"config": endpoint, "converter": converter, "next_time": time(), - "request": requests.request}) + "request": request}) except Exception as e: log.exception(e) diff --git a/thingsboard_gateway/gateway/tb_gateway_service.py b/thingsboard_gateway/gateway/tb_gateway_service.py index bb7cfba5..512dade8 100644 --- a/thingsboard_gateway/gateway/tb_gateway_service.py +++ b/thingsboard_gateway/gateway/tb_gateway_service.py @@ -463,6 +463,7 @@ class TBGatewayService: result = self.__gateway_rpc_methods[method_to_call]() else: result = self.__gateway_rpc_methods[method_to_call]() + log.debug(result) return result def __rpc_ping(self, *args): diff --git a/thingsboard_gateway/tb_utility/tb_utility.py b/thingsboard_gateway/tb_utility/tb_utility.py index 740977e2..e54e4b88 100644 --- a/thingsboard_gateway/tb_utility/tb_utility.py +++ b/thingsboard_gateway/tb_utility/tb_utility.py @@ -140,3 +140,12 @@ class TBUtility: except Exception as e: log.exception(e) return full_value + + @staticmethod + def install_package(package, version="upgrade"): + from sys import executable + from subprocess import check_call + if version.lower() == "upgrade": + check_call([executable, "-m", "pip", "install", package, "--upgrade", "--user"]) + else: + check_call([executable, "-m", "pip", "install", package + version, "--user"]) From 352b51e8fd3da31168c5f6904ff857ba9dcfa857 Mon Sep 17 00:00:00 2001 From: zbeacon Date: Tue, 5 May 2020 11:10:49 +0300 Subject: [PATCH 7/7] File count checking added --- thingsboard_gateway/storage/event_storage_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thingsboard_gateway/storage/event_storage_reader.py b/thingsboard_gateway/storage/event_storage_reader.py index 719f71d9..e921bc74 100644 --- a/thingsboard_gateway/storage/event_storage_reader.py +++ b/thingsboard_gateway/storage/event_storage_reader.py @@ -166,7 +166,7 @@ class EventStorageReader: def delete_read_file(self, current_file: EventStorageReaderPointer): try: data_files = self.files.get_data_files() - if exists(self.settings.get_data_folder_path() + current_file.file): + if exists(self.settings.get_data_folder_path() + current_file.file) and len(data_files) > 1: remove(self.settings.get_data_folder_path() + current_file.file) data_files.remove(current_file.file) log.info("FileStorage_reader -- Cleanup old data file: %s%s!", self.settings.get_data_folder_path(), current_file.file)