From 86479c01f778ac3acd46df4fffb2a3150eae7fd9 Mon Sep 17 00:00:00 2001 From: zbeacon Date: Wed, 24 Jun 2020 12:58:45 +0300 Subject: [PATCH 01/11] Initial commit for SNMP connector --- thingsboard_gateway/connectors/snmp/__init__.py | 0 thingsboard_gateway/extensions/snmp/__init__.py | 15 +++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 thingsboard_gateway/connectors/snmp/__init__.py create mode 100644 thingsboard_gateway/extensions/snmp/__init__.py diff --git a/thingsboard_gateway/connectors/snmp/__init__.py b/thingsboard_gateway/connectors/snmp/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/thingsboard_gateway/extensions/snmp/__init__.py b/thingsboard_gateway/extensions/snmp/__init__.py new file mode 100644 index 00000000..2af4123f --- /dev/null +++ b/thingsboard_gateway/extensions/snmp/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020. ThingsBoard +# # +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# # +# http://www.apache.org/licenses/LICENSE-2.0 +# # +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + From 4d405611fa64283605d06f416eb1723e3d2eb4a4 Mon Sep 17 00:00:00 2001 From: zbeacon Date: Thu, 25 Jun 2020 14:58:51 +0300 Subject: [PATCH 02/11] Basic connector structure SNMp --- .../connectors/snmp/snmp_connector.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 thingsboard_gateway/connectors/snmp/snmp_connector.py diff --git a/thingsboard_gateway/connectors/snmp/snmp_connector.py b/thingsboard_gateway/connectors/snmp/snmp_connector.py new file mode 100644 index 00000000..11cb01a2 --- /dev/null +++ b/thingsboard_gateway/connectors/snmp/snmp_connector.py @@ -0,0 +1,82 @@ +# Copyright 2020. ThingsBoard +# # +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# # +# http://www.apache.org/licenses/LICENSE-2.0 +# # +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from threading import Thread +from time import sleep +from random import choice +from string import ascii_lowercase + +from thingsboard_gateway.connectors.connector import Connector, log +from thingsboard_gateway.tb_utility.tb_utility import TBUtility + +try: + import pysnmp +except ImportError: + TBUtility.install_package("pysnmp") + import pysnmp + + +class SNMPConnector(Connector, Thread): + def __init__(self, gateway, config, connector_type): + super().__init__() + self.daemon = True + self.__gateway = gateway + self._connected = False + self.__stopped = False + self._connector_type = connector_type + self.__config = config + self.setName(config.get("name", 'SNMP Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5)))) + self.statistics = {'MessagesReceived': 0, + 'MessagesSent': 0} + self._default_converters = { + "uplink": "JsonSNMPUplinkConverter", + "downlink": "JsonSNMPDownlinkConverter" + } + + def open(self): + self.__stopped = False + self.start() + + def run(self): + self._connected = True + try: + #some start + while not self.__stopped: + if self.__stopped: + break + else: + sleep(.01) + except Exception as e: + log.exception(e) + + def close(self): + self.__stopped = True + self._connected = False + + def gat_name(self): + return self.name + + def is_connected(self): + return self._connected + + def on_attributes_update(self, content): + log.debug(content) + + def server_side_rpc_handler(self, content): + log.debug(content) + + def collect_statistic_and_send(self, connector_name, data): + self.statistics["MessagesReceived"] = self.statistics["MessagesReceived"] + 1 + self.__gateway.send_to_storage(connector_name, data) + self.statistics["MessagesSent"] = self.statistics["MessagesSent"] + 1 From 18b77c63f9aa55ffdc518ebf2a4d53456711e69b Mon Sep 17 00:00:00 2001 From: zbeacon Date: Tue, 30 Jun 2020 16:05:36 +0300 Subject: [PATCH 03/11] SNMP configuration --- thingsboard_gateway/config/snmp.json | 62 ++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 thingsboard_gateway/config/snmp.json diff --git a/thingsboard_gateway/config/snmp.json b/thingsboard_gateway/config/snmp.json new file mode 100644 index 00000000..88df1a11 --- /dev/null +++ b/thingsboard_gateway/config/snmp.json @@ -0,0 +1,62 @@ +{ + "agent": { + "devices": { + "deviceName": "SNMP router", + "deviceType": "snmp", + "ip": "127.0.0.1", + "community": "public", + "attributes": [ + { + "key": "ReceivedFromGet", + "method": "get", + "oid": "" + }, + { + "key": "ReceivedFromBulkWalk", + "method": "bulkWalk", + "oid": [ + "", + "" + ] + } + ], + "telemetry": [ + { + "key": "ReceivedFromWalk", + "community": "private", + "method": "walk", + "oid": "" + }, + { + "key": "ReceivedFromTable", + "method": "table", + "oid": "" + } + ], + "attributeUpdateRequests": [ + { + "attributeFilter": "dataToSet", + "method": "set", + "oid": "" + } + ], + "serverSideRpcRequests": [ + { + "method": "set", + "oid": "" + }, + { + "method": "get", + "oid": "" + }, + { + "method": "bulkWalk", + "oid": [ + "", + "" + ] + } + ] + } + } +} \ No newline at end of file From bba1b0881bead407cfada94cd9d158475eb4dfa9 Mon Sep 17 00:00:00 2001 From: zbeacon Date: Tue, 30 Jun 2020 16:07:36 +0300 Subject: [PATCH 04/11] SNMP configuration --- thingsboard_gateway/config/snmp.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/thingsboard_gateway/config/snmp.json b/thingsboard_gateway/config/snmp.json index 88df1a11..af59cbce 100644 --- a/thingsboard_gateway/config/snmp.json +++ b/thingsboard_gateway/config/snmp.json @@ -42,14 +42,17 @@ ], "serverSideRpcRequests": [ { + "requestFilter": "setData", "method": "set", "oid": "" }, { + "requestFilter": "getData", "method": "get", "oid": "" }, { + "requestFilter": "runBulkWalk", "method": "bulkWalk", "oid": [ "", From 218d04bea90ca34312d2b13620e91e5838763b28 Mon Sep 17 00:00:00 2001 From: zbeacon Date: Tue, 30 Jun 2020 19:37:43 +0300 Subject: [PATCH 05/11] Improvements --- thingsboard_gateway/config/snmp.json | 27 +++++++++++++---- .../connectors/snmp/snmp_connector.py | 30 ++++++++++++++----- .../connectors/snmp/snmp_uplink_converter.py | 28 +++++++++++++++++ 3 files changed, 71 insertions(+), 14 deletions(-) create mode 100644 thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py diff --git a/thingsboard_gateway/config/snmp.json b/thingsboard_gateway/config/snmp.json index af59cbce..24804ca2 100644 --- a/thingsboard_gateway/config/snmp.json +++ b/thingsboard_gateway/config/snmp.json @@ -1,9 +1,10 @@ { - "agent": { - "devices": { + "devices": [ + { "deviceName": "SNMP router", "deviceType": "snmp", "ip": "127.0.0.1", + "pollPeriod": 5000, "community": "public", "attributes": [ { @@ -13,11 +14,25 @@ }, { "key": "ReceivedFromBulkWalk", - "method": "bulkWalk", + "method": "bulkwalk", "oid": [ "", "" ] + }, + { + "key": "ReceivedFromBulkGet", + "method": "bulkget", + "scalarOid": [ + "", + "" + ], + "repeatingOid": [ + "", + "" + ], + "maxListSize": 10, + "converter": "CustomSNMPConverter" } ], "telemetry": [ @@ -53,7 +68,7 @@ }, { "requestFilter": "runBulkWalk", - "method": "bulkWalk", + "method": "bulkwalk", "oid": [ "", "" @@ -61,5 +76,5 @@ } ] } - } -} \ No newline at end of file + ] +} diff --git a/thingsboard_gateway/connectors/snmp/snmp_connector.py b/thingsboard_gateway/connectors/snmp/snmp_connector.py index 11cb01a2..35383f8d 100644 --- a/thingsboard_gateway/connectors/snmp/snmp_connector.py +++ b/thingsboard_gateway/connectors/snmp/snmp_connector.py @@ -13,7 +13,7 @@ # limitations under the License. from threading import Thread -from time import sleep +from time import sleep, time from random import choice from string import ascii_lowercase @@ -21,10 +21,10 @@ from thingsboard_gateway.connectors.connector import Connector, log from thingsboard_gateway.tb_utility.tb_utility import TBUtility try: - import pysnmp + import puresnmp except ImportError: - TBUtility.install_package("pysnmp") - import pysnmp + TBUtility.install_package("puresnmp") + import puresnmp class SNMPConnector(Connector, Thread): @@ -36,12 +36,22 @@ class SNMPConnector(Connector, Thread): self.__stopped = False self._connector_type = connector_type self.__config = config + self.__devices = self.__config["devices"] self.setName(config.get("name", 'SNMP Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5)))) self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} self._default_converters = { - "uplink": "JsonSNMPUplinkConverter", - "downlink": "JsonSNMPDownlinkConverter" + "uplink": "SNMPUplinkConverter", + "downlink": "SNMPDownlinkConverter" + } + self.__methods = { + "get": puresnmp.get, + "set": puresnmp.set, + "walk": puresnmp.walk, + "table": puresnmp.table, + "bulkGet": puresnmp.bulkget, + "bulkWalk": puresnmp.bulkwalk, + "bulkTable": puresnmp.bulktable, } def open(self): @@ -51,8 +61,12 @@ class SNMPConnector(Connector, Thread): def run(self): self._connected = True try: - #some start while not self.__stopped: + current_time = time()*1000 + for device in self.__devices: + if device.get("previous_poll_time", 0) + device.get("pollPeriod", 10000) < current_time: + + device["previous_poll_time"] = current_time if self.__stopped: break else: @@ -64,7 +78,7 @@ class SNMPConnector(Connector, Thread): self.__stopped = True self._connected = False - def gat_name(self): + def get_name(self): return self.name def is_connected(self): diff --git a/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py b/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py new file mode 100644 index 00000000..2d21affa --- /dev/null +++ b/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py @@ -0,0 +1,28 @@ +# Copyright 2020. ThingsBoard +# # +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# # +# http://www.apache.org/licenses/LICENSE-2.0 +# # +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from thingsboard_gateway.connectors.converter import Converter, log + +class SNMPUplinkConverter(Converter): + def __init__(self, config): + self.__config = config + + def convert(self, config, data): + result = { + "deviceName": self.__config["deviceName"], + "deviceType": self.__config["deviceType"], + "attributes": [], + "telemetry": [] + } + From 7b52309519cef45be3ce43b8e5dd564f0220cf84 Mon Sep 17 00:00:00 2001 From: zbeacon Date: Wed, 1 Jul 2020 10:20:57 +0300 Subject: [PATCH 06/11] Methods processing --- .../connectors/snmp/snmp_connector.py | 125 ++++++++++++++++-- 1 file changed, 113 insertions(+), 12 deletions(-) diff --git a/thingsboard_gateway/connectors/snmp/snmp_connector.py b/thingsboard_gateway/connectors/snmp/snmp_connector.py index 35383f8d..f6eee5ce 100644 --- a/thingsboard_gateway/connectors/snmp/snmp_connector.py +++ b/thingsboard_gateway/connectors/snmp/snmp_connector.py @@ -44,18 +44,12 @@ class SNMPConnector(Connector, Thread): "uplink": "SNMPUplinkConverter", "downlink": "SNMPDownlinkConverter" } - self.__methods = { - "get": puresnmp.get, - "set": puresnmp.set, - "walk": puresnmp.walk, - "table": puresnmp.table, - "bulkGet": puresnmp.bulkget, - "bulkWalk": puresnmp.bulkwalk, - "bulkTable": puresnmp.bulktable, - } + self.__methods = ["get", "multiget", "getnext", "multigetnext", "walk", "multiwalk", "set", "multiset", "bulkget", "bulkwalk", "table", "bulktable"] + self.__datatypes = ('attributes', 'telemetry') def open(self): self.__stopped = False + self.__fill_converters() self.start() def run(self): @@ -64,9 +58,12 @@ class SNMPConnector(Connector, Thread): while not self.__stopped: current_time = time()*1000 for device in self.__devices: - if device.get("previous_poll_time", 0) + device.get("pollPeriod", 10000) < current_time: - - device["previous_poll_time"] = current_time + try: + if device.get("previous_poll_time", 0) + device.get("pollPeriod", 10000) < current_time: + self.__process_data(device) + device["previous_poll_time"] = current_time + except Exception as e: + log.exception(e) if self.__stopped: break else: @@ -94,3 +91,107 @@ class SNMPConnector(Connector, Thread): self.statistics["MessagesReceived"] = self.statistics["MessagesReceived"] + 1 self.__gateway.send_to_storage(connector_name, data) self.statistics["MessagesSent"] = self.statistics["MessagesSent"] + 1 + + def __process_data(self, device): + common_parameters = { + "ip": device["ip"], + "port": device.get("port", 161), + "timeout": device.get("timeout", 6), + "community": device["community"], + } + for datatype in self.__datatypes: + for datatype_config in device[datatype]: + try: + response = None + method = datatype_config.get("method") + if method is None: + log.error("Method not found in configuration: %r", datatype_config) + continue + else: + method = method.lower() + if method not in self.__methods: + log.error("Unknown method: %s, configuration is: %r", method, datatype_config) + + response = self.__process_methods(method, common_parameters, datatype_config) + + except Exception as e: + log.exception(e) + + def __process_methods(self, method, common_parameters, datatype_config): + response = None + + if method == "get": + oid = datatype_config["oid"] + response = puresnmp.get(**common_parameters, + oid=oid) + if method == "multiget": + oids = datatype_config["oid"] + oids = oids if isinstance(oids, list) else list(oids) + response = puresnmp.multiget(**common_parameters, + oids=oids) + if method == "getnext": + oid = datatype_config["oid"] + response = puresnmp.getnext(**common_parameters, + oid=oid) + if method == "multigetnext": + oids = datatype_config["oid"] + oids = oids if isinstance(oids, list) else list(oids) + response = puresnmp.multigetnext(**common_parameters, + oids=oids) + if method == "walk": + oid = datatype_config["oid"] + response = puresnmp.walk(**common_parameters, + oid=oid) + if method == "multiwalk": + oids = datatype_config["oid"] + oids = oids if isinstance(oids, list) else list(oids) + response = puresnmp.multiwalk(**common_parameters, + oids=oids) + if method == "set": + oid = datatype_config["oid"] + value = datatype_config["value"] + response = puresnmp.set(**common_parameters, + oid=oid, + value=value) + if method == "multiset": + mappings = datatype_config["mappings"] + response = puresnmp.multiset(**common_parameters, + mappings=mappings) + if method == "bulkget": + scalar_oids = datatype_config.get("scalarOid", []) + scalar_oids = scalar_oids if isinstance(scalar_oids, list) else list(scalar_oids) + repeating_oids = datatype_config.get("repeatingOid", []) + repeating_oids = repeating_oids if isinstance(repeating_oids, list) else list(repeating_oids) + max_list_size = datatype_config.get("maxListSize", 1) + response = puresnmp.bulkget(**common_parameters, + scalar_oids=scalar_oids, + repeating_oids=repeating_oids, + max_list_size=max_list_size) + if method == "bulkwalk": + oids = datatype_config["oid"] + oids = oids if isinstance(oids, list) else list(oids) + bulk_size = datatype_config.get("bulkSize", 10) + response = puresnmp.bulkwalk(**common_parameters, + bulk_size=bulk_size, + oids=oids) + if method == "table": + oid = datatype_config["oid"] + num_base_nodes = datatype_config.get("numBaseNodes", 0) + response = puresnmp.table(**common_parameters, + oid=oid, + num_base_nodes=num_base_nodes) + if method == "bulktable": + oid = datatype_config["oid"] + num_base_nodes = datatype_config.get("numBaseNodes", 0) + bulk_size = datatype_config.get("bulkSize", 10) + response = puresnmp.bulktable(**common_parameters, + oid=oid, + num_base_nodes=num_base_nodes, + bulk_size=bulk_size) + + return response + + def __fill_converters(self): + for device in self.__devices: + device["uplink_converter"] = TBUtility.check_and_import("snmp", device.get('converter',self._default_converters["uplink"]))(device) + device["downlink_converter"] = TBUtility.check_and_import("snmp", device.get('converter',self._default_converters["downlink"]))(device) From 8d7ac5e76d6a3047f330b48ea21f2751d3a26a34 Mon Sep 17 00:00:00 2001 From: zbeacon Date: Wed, 1 Jul 2020 14:37:50 +0300 Subject: [PATCH 07/11] Improvements for SNMP connector --- thingsboard_gateway/config/snmp.json | 93 +++++++++++++++---- .../connectors/snmp/snmp_connector.py | 34 +++---- .../snmp/snmp_downlink_converter.py | 22 +++++ .../connectors/snmp/snmp_uplink_converter.py | 4 +- .../gateway/tb_gateway_service.py | 1 + 5 files changed, 121 insertions(+), 33 deletions(-) create mode 100644 thingsboard_gateway/connectors/snmp/snmp_downlink_converter.py diff --git a/thingsboard_gateway/config/snmp.json b/thingsboard_gateway/config/snmp.json index 24804ca2..362fe608 100644 --- a/thingsboard_gateway/config/snmp.json +++ b/thingsboard_gateway/config/snmp.json @@ -4,35 +4,62 @@ "deviceName": "SNMP router", "deviceType": "snmp", "ip": "127.0.0.1", + "port": 161, "pollPeriod": 5000, "community": "public", "attributes": [ { "key": "ReceivedFromGet", "method": "get", - "oid": "" + "oid": "1.3.6.0.1.1", + "timeout": 6 + }, + { + "key": "ReceivedFromMultiGet", + "method": "multiget", + "oid": [ + "1.3.6.0.1.1", + "1.3.6.0.2.1" + ], + "timeout": 6 + }, + { + "key": "ReceivedFromGetNext", + "method": "getnext", + "oid": [ + "1.3.6.0.1.1", + "1.3.6.0.2.1" + ], + "timeout": 6 + }, + { + "key": "ReceivedFromMultiWalk", + "method": "multiwalk", + "oid": [ + "1.3.6.0.1.1", + "1.3.6.0.2.1" + ] }, { "key": "ReceivedFromBulkWalk", "method": "bulkwalk", "oid": [ - "", - "" + "1.3.6.0.1.1", + "1.3.6.0.2.1" ] }, { "key": "ReceivedFromBulkGet", "method": "bulkget", "scalarOid": [ - "", - "" + "1.3.6.0.1.1", + "1.3.6.0.2.1" ], "repeatingOid": [ - "", - "" + "1.3.6.0.1.1", + "1.3.6.0.2.1" ], - "maxListSize": 10, - "converter": "CustomSNMPConverter" + "maxListSize": 10 } ], "telemetry": [ @@ -40,41 +67,75 @@ "key": "ReceivedFromWalk", "community": "private", "method": "walk", - "oid": "" + "oid": "1.3.6.0.1.1" }, { "key": "ReceivedFromTable", "method": "table", - "oid": "" + "oid": "1.3.6.0.1.1" } ], "attributeUpdateRequests": [ { "attributeFilter": "dataToSet", "method": "set", - "oid": "" + "oid": "1.3.6.0.1.1" + }, + { + "attributeFilter": "dataToMultiSet", + "method": "multiset", + "mappings": { + "1.2.3": "10", + "2.3.4": "${attribute}" + } } ], "serverSideRpcRequests": [ { "requestFilter": "setData", "method": "set", - "oid": "" + "oid": "1.3.6.0.1.1" + }, + { + "requestFilter": "multiSetData", + "method": "multiset" }, { "requestFilter": "getData", "method": "get", - "oid": "" + "oid": "1.3.6.0.1.1" }, { "requestFilter": "runBulkWalk", "method": "bulkwalk", "oid": [ - "", - "" + "1.3.6.0.1.1", + "1.3.6.0.2.1" ] } ] + }, + { + "deviceName": "SNMP router", + "deviceType": "snmp", + "ip": "127.0.0.1", + "pollPeriod": 5000, + "community": "public", + "converter": "CustomSNMPConverter", + "attributes": [ + { + "key": "ReceivedFromGetWithCustomConverter", + "method": "get", + "oid": "1.3.6.0.1.1" + } + ], + "telemetry": [ + { + "key": "ReceivedFromTableWithCustomConverter", + "method": "table", + "oid": "1.3.6.0.1.1" + } + ] } ] } diff --git a/thingsboard_gateway/connectors/snmp/snmp_connector.py b/thingsboard_gateway/connectors/snmp/snmp_connector.py index f6eee5ce..757f0325 100644 --- a/thingsboard_gateway/connectors/snmp/snmp_connector.py +++ b/thingsboard_gateway/connectors/snmp/snmp_connector.py @@ -111,9 +111,9 @@ class SNMPConnector(Connector, Thread): method = method.lower() if method not in self.__methods: log.error("Unknown method: %s, configuration is: %r", method, datatype_config) - response = self.__process_methods(method, common_parameters, datatype_config) - + converted_data = device["uplink_converter"].convert((datatype, datatype_config), response) + self.collect_statistic_and_send(self.get_name(), converted_data) except Exception as e: log.exception(e) @@ -131,22 +131,24 @@ class SNMPConnector(Connector, Thread): oids=oids) if method == "getnext": oid = datatype_config["oid"] - response = puresnmp.getnext(**common_parameters, - oid=oid) + master_response = puresnmp.getnext(**common_parameters, + oid=oid) + response = {master_response.oid: master_response.value} if method == "multigetnext": oids = datatype_config["oid"] oids = oids if isinstance(oids, list) else list(oids) - response = puresnmp.multigetnext(**common_parameters, - oids=oids) + master_response = puresnmp.multigetnext(**common_parameters, + oids=oids) + response = {binded_var.oid: binded_var.value for binded_var in master_response} if method == "walk": oid = datatype_config["oid"] - response = puresnmp.walk(**common_parameters, - oid=oid) + response = {binded_var.oid: binded_var.value for binded_var in list(puresnmp.walk(**common_parameters, + oid=oid))} if method == "multiwalk": oids = datatype_config["oid"] oids = oids if isinstance(oids, list) else list(oids) - response = puresnmp.multiwalk(**common_parameters, - oids=oids) + response = {binded_var.oid: binded_var.value for binded_var in list(puresnmp.multiwalk(**common_parameters, + oids=oids))} if method == "set": oid = datatype_config["oid"] value = datatype_config["value"] @@ -166,14 +168,14 @@ class SNMPConnector(Connector, Thread): response = puresnmp.bulkget(**common_parameters, scalar_oids=scalar_oids, repeating_oids=repeating_oids, - max_list_size=max_list_size) + max_list_size=max_list_size)._asdict() if method == "bulkwalk": oids = datatype_config["oid"] oids = oids if isinstance(oids, list) else list(oids) bulk_size = datatype_config.get("bulkSize", 10) - response = puresnmp.bulkwalk(**common_parameters, - bulk_size=bulk_size, - oids=oids) + response = {binded_var.oid: binded_var.value for binded_var in list(puresnmp.bulkwalk(**common_parameters, + bulk_size=bulk_size, + oids=oids))} if method == "table": oid = datatype_config["oid"] num_base_nodes = datatype_config.get("numBaseNodes", 0) @@ -193,5 +195,5 @@ class SNMPConnector(Connector, Thread): def __fill_converters(self): for device in self.__devices: - device["uplink_converter"] = TBUtility.check_and_import("snmp", device.get('converter',self._default_converters["uplink"]))(device) - device["downlink_converter"] = TBUtility.check_and_import("snmp", device.get('converter',self._default_converters["downlink"]))(device) + device["uplink_converter"] = TBUtility.check_and_import("snmp", device.get('converter', self._default_converters["uplink"]))(device) + device["downlink_converter"] = TBUtility.check_and_import("snmp", device.get('converter', self._default_converters["downlink"]))(device) diff --git a/thingsboard_gateway/connectors/snmp/snmp_downlink_converter.py b/thingsboard_gateway/connectors/snmp/snmp_downlink_converter.py new file mode 100644 index 00000000..f53f5869 --- /dev/null +++ b/thingsboard_gateway/connectors/snmp/snmp_downlink_converter.py @@ -0,0 +1,22 @@ +# Copyright 2020. ThingsBoard +# # +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# # +# http://www.apache.org/licenses/LICENSE-2.0 +# # +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from thingsboard_gateway.connectors.converter import Converter, log + +class SNMPDownlinkConverter(Converter): + def __init__(self, config): + self.__config = config + + def convert(self, config, data): + return data["params"] diff --git a/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py b/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py index 2d21affa..4d6b2253 100644 --- a/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py +++ b/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py @@ -25,4 +25,6 @@ class SNMPUplinkConverter(Converter): "attributes": [], "telemetry": [] } - + result[config[0]].append({config[1]["key"]: data}) + log.debug(result) + return result diff --git a/thingsboard_gateway/gateway/tb_gateway_service.py b/thingsboard_gateway/gateway/tb_gateway_service.py index 88c8ab35..482911a8 100644 --- a/thingsboard_gateway/gateway/tb_gateway_service.py +++ b/thingsboard_gateway/gateway/tb_gateway_service.py @@ -55,6 +55,7 @@ DEFAULT_CONNECTORS = { "bacnet": "BACnetConnector", "odbc": "OdbcConnector", "rest": "RESTConnector", + "snmp": "SNMPConnector", } class TBGatewayService: From 070bc019c74fdb707023c392b42ea2672c8cac89 Mon Sep 17 00:00:00 2001 From: zbeacon Date: Fri, 3 Jul 2020 15:54:00 +0300 Subject: [PATCH 08/11] Improvements --- thingsboard_gateway/config/snmp.json | 47 +++++++++---------- .../connectors/snmp/snmp_connector.py | 13 +++-- .../connectors/snmp/snmp_uplink_converter.py | 17 ++++++- 3 files changed, 46 insertions(+), 31 deletions(-) diff --git a/thingsboard_gateway/config/snmp.json b/thingsboard_gateway/config/snmp.json index 362fe608..1d782dc6 100644 --- a/thingsboard_gateway/config/snmp.json +++ b/thingsboard_gateway/config/snmp.json @@ -3,7 +3,7 @@ { "deviceName": "SNMP router", "deviceType": "snmp", - "ip": "127.0.0.1", + "ip": "snmp.live.gambitcommunications.com", "port": 161, "pollPeriod": 5000, "community": "public", @@ -11,53 +11,50 @@ { "key": "ReceivedFromGet", "method": "get", - "oid": "1.3.6.0.1.1", + "oid": "1.3.6.1.2.1.1.1.0", "timeout": 6 }, { "key": "ReceivedFromMultiGet", "method": "multiget", "oid": [ - "1.3.6.0.1.1", - "1.3.6.0.2.1" + "1.3.6.1.2.1.1.1.0", + "1.3.6.1.2.1.1.2.0" ], "timeout": 6 }, { "key": "ReceivedFromGetNext", "method": "getnext", - "oid": [ - "1.3.6.0.1.1", - "1.3.6.0.2.1" - ], + "oid": "1.3.6.1.2.1.1.1.0", "timeout": 6 }, { "key": "ReceivedFromMultiWalk", "method": "multiwalk", "oid": [ - "1.3.6.0.1.1", - "1.3.6.0.2.1" + "1.3.6.1.2.1.1.1.0", + "1.3.6.0.1.2.1" ] }, { "key": "ReceivedFromBulkWalk", "method": "bulkwalk", "oid": [ - "1.3.6.0.1.1", - "1.3.6.0.2.1" + "1.3.6.1.2.1.1.1.0", + "1.3.6.1.2.1.1.2.0" ] }, { "key": "ReceivedFromBulkGet", "method": "bulkget", "scalarOid": [ - "1.3.6.0.1.1", - "1.3.6.0.2.1" + "1.3.6.1.2.1.1.1.0", + "1.3.6.1.2.1.1.2.0" ], "repeatingOid": [ - "1.3.6.0.1.1", - "1.3.6.0.2.1" + "1.3.6.1.2.1.1.1.0", + "1.3.6.1.2.1.1.2.0" ], "maxListSize": 10 } @@ -67,19 +64,19 @@ "key": "ReceivedFromWalk", "community": "private", "method": "walk", - "oid": "1.3.6.0.1.1" + "oid": "1.3.6.1.2.1.1.1.0" }, { "key": "ReceivedFromTable", "method": "table", - "oid": "1.3.6.0.1.1" + "oid": "1.3.6.1.2.1.1" } ], "attributeUpdateRequests": [ { "attributeFilter": "dataToSet", "method": "set", - "oid": "1.3.6.0.1.1" + "oid": "1.3.6.1.2.1.1.1.0" }, { "attributeFilter": "dataToMultiSet", @@ -94,7 +91,7 @@ { "requestFilter": "setData", "method": "set", - "oid": "1.3.6.0.1.1" + "oid": "1.3.6.1.2.1.1.1.0" }, { "requestFilter": "multiSetData", @@ -103,14 +100,14 @@ { "requestFilter": "getData", "method": "get", - "oid": "1.3.6.0.1.1" + "oid": "1.3.6.1.2.1.1.1.0" }, { "requestFilter": "runBulkWalk", "method": "bulkwalk", "oid": [ - "1.3.6.0.1.1", - "1.3.6.0.2.1" + "1.3.6.1.2.1.1.1.0", + "1.3.6.1.2.1.1.2.0" ] } ] @@ -126,14 +123,14 @@ { "key": "ReceivedFromGetWithCustomConverter", "method": "get", - "oid": "1.3.6.0.1.1" + "oid": "1.3.6.1.2.1.1.1.0" } ], "telemetry": [ { "key": "ReceivedFromTableWithCustomConverter", "method": "table", - "oid": "1.3.6.0.1.1" + "oid": "1.3.6.1.2.1.1.1.0" } ] } diff --git a/thingsboard_gateway/connectors/snmp/snmp_connector.py b/thingsboard_gateway/connectors/snmp/snmp_connector.py index 757f0325..92ed2b0c 100644 --- a/thingsboard_gateway/connectors/snmp/snmp_connector.py +++ b/thingsboard_gateway/connectors/snmp/snmp_connector.py @@ -16,6 +16,7 @@ from threading import Thread from time import sleep, time from random import choice from string import ascii_lowercase +from socket import gethostbyname from thingsboard_gateway.connectors.connector import Connector, log from thingsboard_gateway.tb_utility.tb_utility import TBUtility @@ -94,7 +95,7 @@ class SNMPConnector(Connector, Thread): def __process_data(self, device): common_parameters = { - "ip": device["ip"], + "ip": gethostbyname(device["ip"]), "port": device.get("port", 161), "timeout": device.get("timeout", 6), "community": device["community"], @@ -178,6 +179,7 @@ class SNMPConnector(Connector, Thread): oids=oids))} if method == "table": oid = datatype_config["oid"] + del common_parameters["timeout"] num_base_nodes = datatype_config.get("numBaseNodes", 0) response = puresnmp.table(**common_parameters, oid=oid, @@ -194,6 +196,9 @@ class SNMPConnector(Connector, Thread): return response def __fill_converters(self): - for device in self.__devices: - device["uplink_converter"] = TBUtility.check_and_import("snmp", device.get('converter', self._default_converters["uplink"]))(device) - device["downlink_converter"] = TBUtility.check_and_import("snmp", device.get('converter', self._default_converters["downlink"]))(device) + try: + for device in self.__devices: + device["uplink_converter"] = TBUtility.check_and_import("snmp", device.get('converter', self._default_converters["uplink"]))(device) + device["downlink_converter"] = TBUtility.check_and_import("snmp", device.get('converter', self._default_converters["downlink"]))(device) + except Exception as e: + log.exception(e) diff --git a/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py b/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py index 4d6b2253..a5a6d0ad 100644 --- a/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py +++ b/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from simplejson import dumps from thingsboard_gateway.connectors.converter import Converter, log @@ -25,6 +26,18 @@ class SNMPUplinkConverter(Converter): "attributes": [], "telemetry": [] } - result[config[0]].append({config[1]["key"]: data}) - log.debug(result) + try: + if isinstance(data, dict): + result[config[0]].append({config[1]["key"]: {str(k): str(v) for k, v in data.items()}}) + elif isinstance(data, list): + if isinstance(data[0], str): + result[config[0]].append({config[1]["key"]: ','.join(data)}) + elif isinstance(data[0], dict): + res = {} + for item in data: + res.update(**item) + result[config[0]].append({config[1]["key"]: {str(k): str(v) for k, v in res.items()}}) + log.debug(result) + except Exception as e: + log.exception(e) return result From 96e7f1f57fec781639922a298c5f5e6ca548a47c Mon Sep 17 00:00:00 2001 From: zbeacon Date: Mon, 13 Jul 2020 15:23:59 +0300 Subject: [PATCH 09/11] SNMP Connector improvements --- thingsboard_gateway/config/tb_gateway.yaml | 5 +++++ .../connectors/snmp/snmp_connector.py | 12 ++++++++++-- .../connectors/snmp/snmp_uplink_converter.py | 5 ++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/thingsboard_gateway/config/tb_gateway.yaml b/thingsboard_gateway/config/tb_gateway.yaml index cc66a24e..ff94c165 100644 --- a/thingsboard_gateway/config/tb_gateway.yaml +++ b/thingsboard_gateway/config/tb_gateway.yaml @@ -65,6 +65,11 @@ connectors: # configuration: rest.json # # - +# name: SNMP Connector +# type: snmp +# configuration: snmp.json +# +# - # name: Custom Serial Connector # type: serial # configuration: custom_serial.json diff --git a/thingsboard_gateway/connectors/snmp/snmp_connector.py b/thingsboard_gateway/connectors/snmp/snmp_connector.py index 92ed2b0c..9d7a86b1 100644 --- a/thingsboard_gateway/connectors/snmp/snmp_connector.py +++ b/thingsboard_gateway/connectors/snmp/snmp_connector.py @@ -27,6 +27,8 @@ except ImportError: TBUtility.install_package("puresnmp") import puresnmp +from puresnmp.exc import Timeout as SNMPTimeoutException + class SNMPConnector(Connector, Thread): def __init__(self, gateway, config, connector_type): @@ -100,6 +102,7 @@ class SNMPConnector(Connector, Thread): "timeout": device.get("timeout", 6), "community": device["community"], } + converted_data = {} for datatype in self.__datatypes: for datatype_config in device[datatype]: try: @@ -113,10 +116,15 @@ class SNMPConnector(Connector, Thread): if method not in self.__methods: log.error("Unknown method: %s, configuration is: %r", method, datatype_config) response = self.__process_methods(method, common_parameters, datatype_config) - converted_data = device["uplink_converter"].convert((datatype, datatype_config), response) - self.collect_statistic_and_send(self.get_name(), converted_data) + converted_data.update(**device["uplink_converter"].convert((datatype, datatype_config), response)) + except SNMPTimeoutException: + log.error("Timeout exception on connection to device \"%s\" with ip: \"%s\"", device["deviceName"], device["ip"]) + return except Exception as e: log.exception(e) + if isinstance(converted_data, dict) and (converted_data.get("attributes") or converted_data.get("telemetry")): + self.collect_statistic_and_send(self.get_name(), converted_data) + def __process_methods(self, method, common_parameters, datatype_config): response = None diff --git a/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py b/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py index a5a6d0ad..8b31b804 100644 --- a/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py +++ b/thingsboard_gateway/connectors/snmp/snmp_uplink_converter.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from simplejson import dumps from thingsboard_gateway.connectors.converter import Converter, log @@ -37,6 +36,10 @@ class SNMPUplinkConverter(Converter): for item in data: res.update(**item) result[config[0]].append({config[1]["key"]: {str(k): str(v) for k, v in res.items()}}) + elif isinstance(data, str): + result[config[0]].append({config[1]["key"]: data}) + elif isinstance(data, bytes): + result[config[0]].append({config[1]["key"]: data.decode("UTF-8")}) log.debug(result) except Exception as e: log.exception(e) From 8d74801d2d416992bc01de87d99cc88fcc1dcc98 Mon Sep 17 00:00:00 2001 From: zbeacon Date: Mon, 13 Jul 2020 16:19:37 +0300 Subject: [PATCH 10/11] SNMP Connector added attribute update processing --- .../connectors/snmp/snmp_connector.py | 52 ++++++++++++------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/thingsboard_gateway/connectors/snmp/snmp_connector.py b/thingsboard_gateway/connectors/snmp/snmp_connector.py index 9d7a86b1..31a591c0 100644 --- a/thingsboard_gateway/connectors/snmp/snmp_connector.py +++ b/thingsboard_gateway/connectors/snmp/snmp_connector.py @@ -85,7 +85,16 @@ class SNMPConnector(Connector, Thread): return self._connected def on_attributes_update(self, content): - log.debug(content) + for device in self.__devices: + if content["device"] == device["deviceName"]: + for attribute_request_config in device["attributeUpdateRequests"]: + for attribute, value in content["data"]: + if attribute == TBUtility.get_value(attribute_request_config["attributeFilter"], "", get_tag=True): + common_parameters = self.__get_common_parameters(device) + result = self.__process_methods(attribute_request_config["method"], common_parameters, {**attribute_request_config, "value": value}) + log.debug("Received attribute update request for device \"%s\" with attribute \"%s\" and value \"%s\"", content["device"], attribute) + + log.debug(content) def server_side_rpc_handler(self, content): log.debug(content) @@ -96,12 +105,7 @@ class SNMPConnector(Connector, Thread): self.statistics["MessagesSent"] = self.statistics["MessagesSent"] + 1 def __process_data(self, device): - common_parameters = { - "ip": gethostbyname(device["ip"]), - "port": device.get("port", 161), - "timeout": device.get("timeout", 6), - "community": device["community"], - } + common_parameters = self.__get_common_parameters(device) converted_data = {} for datatype in self.__datatypes: for datatype_config in device[datatype]: @@ -125,7 +129,6 @@ class SNMPConnector(Connector, Thread): if isinstance(converted_data, dict) and (converted_data.get("attributes") or converted_data.get("telemetry")): self.collect_statistic_and_send(self.get_name(), converted_data) - def __process_methods(self, method, common_parameters, datatype_config): response = None @@ -133,42 +136,42 @@ class SNMPConnector(Connector, Thread): oid = datatype_config["oid"] response = puresnmp.get(**common_parameters, oid=oid) - if method == "multiget": + elif method == "multiget": oids = datatype_config["oid"] oids = oids if isinstance(oids, list) else list(oids) response = puresnmp.multiget(**common_parameters, oids=oids) - if method == "getnext": + elif method == "getnext": oid = datatype_config["oid"] master_response = puresnmp.getnext(**common_parameters, oid=oid) response = {master_response.oid: master_response.value} - if method == "multigetnext": + elif method == "multigetnext": oids = datatype_config["oid"] oids = oids if isinstance(oids, list) else list(oids) master_response = puresnmp.multigetnext(**common_parameters, oids=oids) response = {binded_var.oid: binded_var.value for binded_var in master_response} - if method == "walk": + elif method == "walk": oid = datatype_config["oid"] response = {binded_var.oid: binded_var.value for binded_var in list(puresnmp.walk(**common_parameters, oid=oid))} - if method == "multiwalk": + elif method == "multiwalk": oids = datatype_config["oid"] oids = oids if isinstance(oids, list) else list(oids) response = {binded_var.oid: binded_var.value for binded_var in list(puresnmp.multiwalk(**common_parameters, oids=oids))} - if method == "set": + elif method == "set": oid = datatype_config["oid"] value = datatype_config["value"] response = puresnmp.set(**common_parameters, oid=oid, value=value) - if method == "multiset": + elif method == "multiset": mappings = datatype_config["mappings"] response = puresnmp.multiset(**common_parameters, mappings=mappings) - if method == "bulkget": + elif method == "bulkget": scalar_oids = datatype_config.get("scalarOid", []) scalar_oids = scalar_oids if isinstance(scalar_oids, list) else list(scalar_oids) repeating_oids = datatype_config.get("repeatingOid", []) @@ -178,21 +181,21 @@ class SNMPConnector(Connector, Thread): scalar_oids=scalar_oids, repeating_oids=repeating_oids, max_list_size=max_list_size)._asdict() - if method == "bulkwalk": + elif method == "bulkwalk": oids = datatype_config["oid"] oids = oids if isinstance(oids, list) else list(oids) bulk_size = datatype_config.get("bulkSize", 10) response = {binded_var.oid: binded_var.value for binded_var in list(puresnmp.bulkwalk(**common_parameters, bulk_size=bulk_size, oids=oids))} - if method == "table": + elif method == "table": oid = datatype_config["oid"] del common_parameters["timeout"] num_base_nodes = datatype_config.get("numBaseNodes", 0) response = puresnmp.table(**common_parameters, oid=oid, num_base_nodes=num_base_nodes) - if method == "bulktable": + elif method == "bulktable": oid = datatype_config["oid"] num_base_nodes = datatype_config.get("numBaseNodes", 0) bulk_size = datatype_config.get("bulkSize", 10) @@ -200,7 +203,8 @@ class SNMPConnector(Connector, Thread): oid=oid, num_base_nodes=num_base_nodes, bulk_size=bulk_size) - + else: + log.error("Method \"%s\" - Not found", str(method)) return response def __fill_converters(self): @@ -210,3 +214,11 @@ class SNMPConnector(Connector, Thread): device["downlink_converter"] = TBUtility.check_and_import("snmp", device.get('converter', self._default_converters["downlink"]))(device) except Exception as e: log.exception(e) + + @staticmethod + def __get_common_parameters(device): + return {"ip": gethostbyname(device["ip"]), + "port": device.get("port", 161), + "timeout": device.get("timeout", 6), + "community": device["community"], + } From 00663d304c4c2e92431657eb901a649ff5f4d2b5 Mon Sep 17 00:00:00 2001 From: zbeacon Date: Tue, 14 Jul 2020 09:24:36 +0300 Subject: [PATCH 11/11] Added RPC call processing --- .../connectors/snmp/snmp_connector.py | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/thingsboard_gateway/connectors/snmp/snmp_connector.py b/thingsboard_gateway/connectors/snmp/snmp_connector.py index 31a591c0..e921d848 100644 --- a/thingsboard_gateway/connectors/snmp/snmp_connector.py +++ b/thingsboard_gateway/connectors/snmp/snmp_connector.py @@ -17,6 +17,7 @@ from time import sleep, time from random import choice from string import ascii_lowercase from socket import gethostbyname +from re import search from thingsboard_gateway.connectors.connector import Connector, log from thingsboard_gateway.tb_utility.tb_utility import TBUtility @@ -85,19 +86,35 @@ class SNMPConnector(Connector, Thread): return self._connected def on_attributes_update(self, content): - for device in self.__devices: - if content["device"] == device["deviceName"]: - for attribute_request_config in device["attributeUpdateRequests"]: - for attribute, value in content["data"]: - if attribute == TBUtility.get_value(attribute_request_config["attributeFilter"], "", get_tag=True): - common_parameters = self.__get_common_parameters(device) - result = self.__process_methods(attribute_request_config["method"], common_parameters, {**attribute_request_config, "value": value}) - log.debug("Received attribute update request for device \"%s\" with attribute \"%s\" and value \"%s\"", content["device"], attribute) - - log.debug(content) + try: + for device in self.__devices: + if content["device"] == device["deviceName"]: + for attribute_request_config in device["attributeUpdateRequests"]: + for attribute, value in content["data"]: + if search(attribute, attribute_request_config["attributeFilter"]): + common_parameters = self.__get_common_parameters(device) + result = self.__process_methods(attribute_request_config["method"], common_parameters, {**attribute_request_config, "value": value}) + log.debug("Received attribute update request for device \"%s\" with attribute \"%s\" and value \"%s\"", content["device"], attribute) + log.debug(result) + log.debug(content) + except Exception as e: + log.exception(e) def server_side_rpc_handler(self, content): - log.debug(content) + try: + for device in self.__devices: + if content["device"] == device["deviceName"]: + for rpc_request_config in device["serverSideRpcRequests"]: + if search(content["data"]["method"], rpc_request_config["requestFilter"]): + common_parameters = self.__get_common_parameters(device) + result = self.__process_methods(rpc_request_config["method"], common_parameters, {**rpc_request_config, "value": content["data"]["params"]}) + log.debug("Received RPC request for device \"%s\" with command \"%s\" and value \"%s\"", content["device"], content["data"]["method"]) + log.debug(result) + log.debug(content) + self.__gateway.send_rpc_reply(device=content["device"], req_id=content["data"]["id"], content=result) + except Exception as e: + log.exception(e) + self.__gateway.send_rpc_reply(device=content["device"], req_id=content["data"]["id"], success_sent=False) def collect_statistic_and_send(self, connector_name, data): self.statistics["MessagesReceived"] = self.statistics["MessagesReceived"] + 1 @@ -129,7 +146,8 @@ class SNMPConnector(Connector, Thread): if isinstance(converted_data, dict) and (converted_data.get("attributes") or converted_data.get("telemetry")): self.collect_statistic_and_send(self.get_name(), converted_data) - def __process_methods(self, method, common_parameters, datatype_config): + @staticmethod + def __process_methods(method, common_parameters, datatype_config): response = None if method == "get":