mirror/thingsboard-gateway
1
0
Fork 0
mirror of https://github.com/thingsboard/thingsboard-gateway synced 2025-10-26 22:31:42 +08:00
Code Issues Releases Wiki Activity

Merge branch 'thingsboard:feature/sql-lite-storage' into feature/sql-lite-storage

Browse Source
This commit is contained in:
Vitalii 2021-09-28 15:49:04 +03:00 committed by GitHub
commit daa865d644
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 121 additions and 269 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
thingsboard_gateway/config/connected_devices.json Normal file
View File

@ -0,0 +1,9 @@
{
"ODBC 9D36A330-0BE6-11EC-A521-7560092E757E": "ODBC Connector",
"ODBC customer@thingsboard.org": "ODBC Connector1",
"ODBC customerA@thingsboard.org": "ODBC Connector1",
"ODBC customerB@thingsboard.org": "ODBC Connector1",
"ODBC customerC@thingsboard.org": "ODBC Connector1",
"ODBC sysadmin@thingsboard.org": "ODBC Connector1",
"ODBC tenant@thingsboard.org": "ODBC Connector1"
}

24
thingsboard_gateway/config/logs.conf
View File

@ -1,54 +1,54 @@
[loggers]
keys=root, service, connector, converter, tb_connection, storage, extension, database
keys=root, service, connector, converter, tb_connection, storage, extension
[handlers]
keys=consoleHandler, serviceHandler, connectorHandler, converterHandler, tb_connectionHandler, storageHandler, extensionHandler, databaseHandler
keys=consoleHandler, serviceHandler, connectorHandler, converterHandler, tb_connectionHandler, storageHandler, extensionHandler
[formatters]
keys=LogFormatter
[logger_root]
level=INFO
level=ERROR
handlers=consoleHandler
[logger_connector]
level=DEBUG
level=INFO
handlers=connectorHandler
formatter=LogFormatter
qualname=connector
[logger_storage]
level=DEBUG
level=INFO
handlers=storageHandler
formatter=LogFormatter
qualname=storage
[logger_database]
level=DEBUG
level=INFO
handlers=databaseHandler
formatter=LogFormatter
qualname=database
[logger_tb_connection]
level=DEBUG
level=INFO
handlers=tb_connectionHandler
formatter=LogFormatter
qualname=tb_connection
[logger_service]
level=DEBUG
level=INFO
handlers=serviceHandler
formatter=LogFormatter
qualname=service
[logger_converter]
level=DEBUG
level=INFO
handlers=converterHandler
formatter=LogFormatter
qualname=converter
[logger_extension]
level=DEBUG
level=INFO
handlers=connectorHandler
formatter=LogFormatter
qualname=extension
[handler_consoleHandler]
class=StreamHandler
level=DEBUG
level=INFO
formatter=LogFormatter
args=(sys.stdout,)
[handler_connectorHandler]
level=DEBUG
level=INFO
class=logging.handlers.TimedRotatingFileHandler
formatter=LogFormatter
args=("./logs/connector.log", "d", 1, 7,)

2
thingsboard_gateway/config/modbus.json
View File

@ -169,4 +169,4 @@
}
]
}
}
}

58
thingsboard_gateway/config/odbc1.json Normal file
View File

@ -0,0 +1,58 @@
{
"connection": {
"str": "Driver={PostgreSQL};Server=localhost;Port=5432;Database=thingsboard;Uid=postgres;Pwd=postgres;",
"attributes": {
"autocommit": true,
"timeout": 0
},
"encoding": "utf-8",
"decoding": {
"char": "utf-8",
"wchar": "utf-8",
"metadata": "utf-16le"
},
"reconnect": true,
"reconnectPeriod": 60
},
"pyodbc": {
"pooling": false
},
"polling": {
"query": "SELECT * FROM tb_user WHERE created_time > ? ORDER BY created_time ASC LIMIT 10",
"period": 10,
"iterator": {
"column": "created_time",
"query": "SELECT MIN(created_time) - 1 FROM tb_user",
"persistent": false
}
},
"mapping": {
"device": {
"type": "postgres",
"name": "'ODBC ' + email"
},
"sendDataOnlyOnChange": false,
"attributes": "*",
"timeseries": [
{
"name": "value",
"value": "[i for i in additional_info if i is not None][0]"
}
]
},
"serverSideRpc": {
"enableUnknownRpc": false,
"overrideRpcConfig": true,
"methods": [
"procedureOne",
{
"name": "procedureTwo",
"args": [
"One",
2,
3.0
]
}
]
}
}

6
thingsboard_gateway/config/tb-cloud-chain.pem Normal file
View File

@ -0,0 +1,6 @@
-----BEGIN CERTIFICATE-----
MIIFFjCCAv6gAwIBAgIRAJErCErPDBinU/bWLiWnX1owDQYJKoZIhvcNAQELBQAwTzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2VhcmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMjAwOTA0MDAwMDAwWhcNMjUwOTE1MTYwMDAwWjAyMQswCQYDVQQGEwJVUzEWMBQGA1UEChMNTGV0J3MgRW5jcnlwdDELMAkGA1UEAxMCUjMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC7AhUozPaglNMPEuyNVZLD+ILxmaZ6QoinXSaqtSu5xUyxr45r+XXIo9cPR5QUVTVXjJ6oojkZ9YI8QqlObvU7wy7bjcCwXPNZOOftz2nwWgsbvsCUJCWH+jdxsxPnHKzhm+/b5DtFUkWWqcFTzjTIUu61ru2P3mBw4qVUq7ZtDpelQDRrK9O8ZutmNHz6a4uPVymZ+DAXXbpyb/uBxa3Shlg9F8fnCbvxK/eG3MHacV3URuPMrSXBiLxgZ3Vms/EY96Jc5lP/Ooi2R6X/ExjqmAl3P51T+c8B5fWmcBcUr2Ok/5mzk53cU6cG/kiFHaFpriV1uxPMUgP17VGhi9sVAgMBAAGjggEIMIIBBDAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFBQusxe3WFbLrlAJQOYfr52LFMLGMB8GA1UdIwQYMBaAFHm0WeZ7tuXkAXOACIjIGlj26ZtuMDIGCCsGAQUFBwEBBCYwJDAiBggrBgEFBQcwAoYWaHR0cDovL3gxLmkubGVuY3Iub3JnLzAnBgNVHR8EIDAeMBygGqAYhhZodHRwOi8veDEuYy5sZW5jci5vcmcvMCIGA1UdIAQbMBkwCAYGZ4EMAQIBMA0GCysGAQQBgt8TAQEBMA0GCSqGSIb3DQEBCwUAA4ICAQCFyk5HPqP3hUSFvNVneLKYY611TR6WPTNlclQtgaDqw+34IL9fzLdwALduO/ZelN7kIJ+m74uyA+eitRY8kc607TkC53wlikfmZW4/RvTZ8M6UK+5UzhK8jCdLuMGYL6KvzXGRSgi3yLgjewQtCPkIVz6D2QQzCkcheAmCJ8MqyJu5zlzyZMjAvnnAT45tRAxekrsu94sQ4egdRCnbWSDtY7kh+BImlJNXoB1lBMEKIq4QDUOXoRgffuDghje1WrG9ML+Hbisq/yFOGwXD9RiX8F6sw6W4avAuvDszue5L3sz85K+EC4Y/wFVDNvZo4TYXao6Z0f+lQKc0t8DQYzk1OXVu8rp2yJMC6alLbBfODALZvYH7n7do1AZls4I9d1P4jnkDrQoxB3UqQ9hVl3LEKQ73xF1OyK5GhDDX8oVfGKF5u+decIsH4YaTw7mP3GFxJSqv3+0lUFJoi5Lc5da149p90IdshCExroL1+7mryIkXPeFM5TgO9r0rvZaBFOvV2z0gp35Z0+L4WPlbuEjN/lxPFin+HlUjr8gRsI3qfJOQFy/9rKIJR0Y/8Omwt/8oTWgy1mdeHmmjk7j1nYsvC9JSQ6ZvMldlTTKB3zhThV1+XWYp6rjd5JW1zbVWEkLNxE7GJThEUG3szgBVGP7pSWTUTsqXnLRbwHOoq7hHwg==
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAwTzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2VhcmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJuZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBYMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygch77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6UA5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sWT8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyHB5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UCB5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUvKBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWnOlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTnjh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbwqHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CIrU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkqhkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZLubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KKNFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7UrTkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdCjNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVcoyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPAmRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57demyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
-----END CERTIFICATE-----

148
thingsboard_gateway/extensions/mqtt/shelly_converter.py
View File

@ -1,148 +0,0 @@
# Copyright 2020. ThingsBoard
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from simplejson import dumps
from thingsboard_gateway.connectors.mqtt.mqtt_uplink_converter import MqttUplinkConverter, log
import re
class ShellyConverter(MqttUplinkConverter):
def __init__(self, config):
self.__config = config.get('converter')
self.dict_result = {}
def convert(self, topic, body):
log.info("DATA: " + str(body) + ", Topic: " + topic + ", Config:" + dumps(self.__config))
# Search the device name from the topic
#prog = re.compile(self.__config["deviceNameTopicExpression"])
topicSplit = topic.split("/")
deviceName = topicSplit[1]
deviceConverter = deviceName.split("-")[0]
deviceMac = deviceName.split("-")[1]
try:
# Shellyht
if deviceConverter == "shellyht":
log.debug("New Message! Selecting converter: %s" % deviceConverter)
# Get list of only the telemetry we want to post
#telemetryKeys = self.__config["extension-config"].keys()
# Get last identifier from topic to be able to compare against telemetryKeys
_topicTelemetry = topic.split("/")[-1]
self.dict_result["attributes"] = []
self.dict_result["attributes"].append({"MAC": deviceMac})
self.dict_result["telemetry"] = [] # Telemetry template
self.dict_result["deviceType"] = self.__config["deviceTypeTopicExpression"] # pass from config
self.dict_result["deviceName"] = deviceName # Get the full string
self.dict_result["telemetry"].append({ _topicTelemetry : str(body) })
log.info(dumps(self.dict_result))
return self.dict_result
# Shellyem3 and Shellyem
elif deviceConverter == "shellyem3" or deviceConverter == "shellyem":
log.debug("New Message! Selecting converter: %s" % deviceConverter)
_phase = topicSplit[3]
_elType = topicSplit[-1]
self.dict_result["attributes"] = []
self.dict_result["attributes"].append({"MAC": deviceMac})
self.dict_result["telemetry"] = []
self.dict_result["deviceType"] = self.__config["deviceTypeTopicExpression"]
self.dict_result["deviceName"] = deviceName
if "relay" in topicSplit:
tel = {("Relay" + str(_phase)) : str(body)}
self.dict_result["telemetry"].append(tel)
return self.dict_result
self.dict_result["telemetry"].append({("L" + str(int(_phase) + 1) + "_") + str(_elType) : str(body) })
log.info(dumps(self.dict_result))
return self.dict_result
# Shelly1pm
elif deviceConverter == "shelly1pm":
log.debug("New Message! Selecting converter: %s" % deviceConverter)
self.dict_result["attributes"] = []
self.dict_result["attributes"].append({"MAC": deviceMac})
self.dict_result["telemetry"] = []
self.dict_result["deviceType"] = self.__config["deviceTypeTopicExpression"]
self.dict_result["deviceName"] = deviceName
key = ""
if "temperature" == topicSplit[-1].split('_')[0]:
_key = "deviceTemperature"
self.dict_result["telemetry"].append({_key : str(body)})
return self.dict_result
_phase = topicSplit[3]
_elType = topicSplit[-1]
if str(topicSplit[-1]) == "0":
telemetry = {("Relay" + str(_elType)) : str(body)}
self.dict_result["telemetry"].append(telemetry)
return self.dict_result
telemetry = {("L" + str(int(_phase) + 1) + "_") + str(_elType) : str(body)}
self.dict_result["telemetry"].append(telemetry)
return self.dict_result
# Shelly2.5
elif deviceConverter == "shellyswitch25":
log.debug("New Message! Selecting converter: %s" % deviceConverter)
self.dict_result["attributes"] = []
self.dict_result["attributes"].append({"MAC": deviceMac})
self.dict_result["telemetry"] = []
self.dict_result["deviceType"] = self.__config["deviceTypeTopicExpression"]
self.dict_result["deviceName"] = deviceName
if "temperature" == topicSplit[-1].split('_')[0]:
_key = "deviceTemperature"
self.dict_result["telemetry"].append({_key : str(body)})
return self.dict_result
_phase = topicSplit[3]
_elType = topicSplit[-1]
if len(topicSplit[-1]) < 2:
telemetry = {("Relay" + str(_elType)) : str(body)}
self.dict_result["telemetry"].append(telemetry)
return self.dict_result
telemetry = {("L" + str(int(_phase) + 1) + "_") + str(_elType) : str(body)}
self.dict_result["telemetry"].append(telemetry)
return self.dict_result
except Exception as e:
log.exception('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), body)
log.exception(e)

94
thingsboard_gateway/extensions/mqtt/zb_tasmota_converter.py
View File

@ -1,94 +0,0 @@
# Copyright 2020. ThingsBoard
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from simplejson import dumps, loads
from thingsboard_gateway.connectors.mqtt.mqtt_uplink_converter import MqttUplinkConverter, log
from thingsboard_gateway.tb_utility.tb_utility import TBUtility
from jsonpath_rw import parse
import re
class ZbTasmotaConverter(MqttUplinkConverter):
def __init__(self, config):
self.__config = config.get('converter')
self.dict_result = {}
def convert(self, topic, body):
log.info("DATA: " + str(body) + ", Topic: " + topic + ", Config:" + dumps(self.__config))
# MAP
# This map is neccessary in order to have some protocol on what device is what
# Every Zigbee Device name should start with single byte integer eg: 0_Kitchen_TempHumi
# The integer will be stored as string becouse of .split() func
deviceMap = {
"0": "ZigbeeButton",
"1": "ZigbeeTempHumi"
}
body = loads(dumps(body)) # We are expecting JSON input
# get device name and device id and device Data
# NOTE: this parses the mqtt.json Json Expressions and looks for the name in message
deviceName = TBUtility.get_value(self.__config["deviceNameJsonExpression"], body)
deviceData = body[deviceName]
deviceConverter = deviceName.split("_")[0]
try:
self.dict_result["attributes"] = [] # Attributes template
self.dict_result["telemetry"] = [] # Telemetry template
# ZigbeeButton
if deviceConverter == "0":
self.dict_result["deviceType"] = deviceMap[deviceConverter]
self.dict_result["deviceName"] = deviceName
# Telemetry
self.dict_result["telemetry"].append({"linkQuality": str(deviceData["LinkQuality"])})
if deviceData.get("Power"):
self.dict_result["telemetry"].append({"buttonPress": str(deviceData["Power"])})
# Attributes
self.dict_result["attributes"].append({"zigbeeShortAddress": str(deviceData["Device"])}) # Always in the message
if deviceData.get("BatteryPercentage"):
self.dict_result["attributes"].append({"batteryPercentage": str(deviceData["BatteryPercentage"])})
return self.dict_result
# ZigbeeTempHumi
if deviceConverter == "1":
self.dict_result["deviceType"] = deviceMap[deviceConverter]
self.dict_result["deviceName"] = deviceName
# Telemetry
self.dict_result["telemetry"].append({"linkQuality": str(deviceData["LinkQuality"])})
if deviceData.get("Temperature"):
self.dict_result["telemetry"].append({"temperature": str(deviceData["Temperature"])})
if deviceData.get("Humidity"):
self.dict_result["telemetry"].append({"humidity": str(deviceData["Humidity"])})
# Attributes
self.dict_result["attributes"].append({"zigbeeShortAddress": str(deviceData["Device"])})
if deviceData.get("BatteryPercentage"):
self.dict_result["attributes"].append({"batteryPercentage": str(deviceData["BatteryPercentage"])})
return self.dict_result
log.debug(dumps(self.dict_result))
except KeyError as ke:
log.warning("Key was not supplied: %s" % str(ke))
except Exception as e:
log.exception('Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), body)
log.exception(e)

2
thingsboard_gateway/gateway/tb_gateway_service.py
View File

@ -437,7 +437,7 @@ class TBGatewayService:
# disconnected = False
# last_disconnect = None
while True:
while not self.stopped:
try:
if self.tb_client.is_connected():
size = getsizeof(devices_data_in_event_pack)

5
thingsboard_gateway/storage/event_storage.py
View File

@ -33,3 +33,8 @@ class EventStorage(ABC):
def event_pack_processing_done(self):
# Indicates that events from previous "get_event_pack" may be cleared
pass
@abstractmethod
def stop(self):
# Stop the storage processing
pass

25
thingsboard_gateway/storage/file/file_event_storage.py
View File

@ -33,17 +33,22 @@ class FileEventStorage(EventStorage):
self.state_file = self.event_storage_files.get_state_file()
self.__writer = EventStorageWriter(self.event_storage_files, self.settings)
self.__reader = EventStorageReader(self.event_storage_files, self.settings)
self.__stopped = False
def put(self, event):
try:
self.__writer.write(event)
except DataFileCountError as e:
log.error(e)
except Exception as e:
log.exception(e)
return False
success = False
if not self.__stopped:
try:
self.__writer.write(event)
except DataFileCountError as e:
log.error(e)
except Exception as e:
log.exception(e)
else:
success = True
else:
return True
log.error("Storage is closed!")
return success
def get_event_pack(self):
return self.__reader.read()
@ -92,3 +97,7 @@ class FileEventStorage(EventStorage):
return prefix + filename + '.txt'
except IOError as e:
log.error("Failed to create a new file! Error: %s", e)
def stop(self):
self.__stopped = True

17
thingsboard_gateway/storage/memory/memory_event_storage.py
View File

@ -23,16 +23,20 @@ class MemoryEventStorage(EventStorage):
self.__events_per_time = config.get("read_records_count", 1000)
self.__events_queue = Queue(self.__queue_len)
self.__event_pack = []
self.__stopped = False
log.debug("Memory storage created with following configuration: \nMax size: %i\n Read records per time: %i",
self.__queue_len, self.__events_per_time)
def put(self, event):
success = False
try:
self.__events_queue.put(event)
success = True
except Full:
log.error("Memory storage is full!")
if not self.__stopped:
try:
self.__events_queue.put(event)
success = True
except Full:
log.error("Memory storage is full!")
else:
log.error("Storage is stopped!")
return success
def get_event_pack(self):
@ -46,3 +50,6 @@ class MemoryEventStorage(EventStorage):
def event_pack_processing_done(self):
self.__event_pack = []
def stop(self):
self.__stopped = True