mirror/thingsboard-gateway
1
0
Fork 0
mirror of https://github.com/thingsboard/thingsboard-gateway synced 2025-10-26 22:31:42 +08:00
Code Issues Releases Wiki Activity

Merge pull request #1107 from samson0v/master

Browse Source
This commit is contained in:
Illia Barkov
2023-05-02 13:56:58 +03:00
committed by GitHub
parent bd7f539961 cc89caed09
commit a8d9e312cb
3 changed files with 69 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
thingsboard_gateway/connectors/ftp/ftp_connector.py
View File

@@ -143,7 +143,7 @@ class FTPConnector(Connector, Thread):
for file in path.files:
current_hash = file.get_current_hash(ftp)
if ((file.has_hash() and current_hash != file.hash)
or not file.has_hash()) and file.check_size_limit(ftp):
or not file.has_hash()) and file.check_size_limit(ftp):
file.set_new_hash(current_hash)
handle_stream = io.BytesIO()
@@ -160,14 +160,10 @@ class FTPConnector(Connector, Thread):
if isinstance(json_data, list):
for obj in json_data:
converted_data = converter.convert(convert_conf, obj)
self.__gateway.send_to_storage(self.getName(), converted_data)
self.statistics['MessagesSent'] = self.statistics['MessagesSent'] + 1
log.debug("Data to ThingsBoard: %s", converted_data)
self.__send_data(converted_data)
else:
converted_data = converter.convert(convert_conf, json_data)
self.__gateway.send_to_storage(self.getName(), converted_data)
self.statistics['MessagesSent'] = self.statistics['MessagesSent'] + 1
log.debug("Data to ThingsBoard: %s", converted_data)
self.__send_data(converted_data)
else:
cursor = file.cursor or 0
@@ -182,12 +178,16 @@ class FTPConnector(Connector, Thread):
else:
converted_data = converter.convert(convert_conf, line)
self.__gateway.send_to_storage(self.getName(), converted_data)
self.statistics['MessagesSent'] = self.statistics['MessagesSent'] + 1
log.debug("Data to ThingsBoard: %s", converted_data)
self.__send_data(converted_data)
handle_stream.close()
def __send_data(self, converted_data):
if converted_data:
self.__gateway.send_to_storage(self.getName(), converted_data)
self.statistics['MessagesSent'] = self.statistics['MessagesSent'] + 1
log.debug("Data to ThingsBoard: %s", converted_data)
def close(self):
self.__stopped = True

19
thingsboard_gateway/connectors/ftp/ftp_uplink_converter.py
View File

@@ -196,12 +196,13 @@ class FTPUplinkConverter(FTPConverter):
@StatisticsService.CollectStatistics(start_stat_type='receivedBytesFromDevices',
end_stat_type='convertedBytesFromDevice')
def convert(self, config, data):
if config['file_ext'] == 'csv' or (
config['file_ext'] == 'txt' and self.__config['txt_file_data_view'] == 'TABLE'):
return self._convert_table_view_data(config, data)
elif config['file_ext'] == 'txt' and self.__config['txt_file_data_view'] == 'SLICED':
return self._convert_slices_view_data(data)
elif config['file_ext'] == 'json':
return self._convert_json_file(data)
else:
raise Exception('Incorrect file extension or file data view mode')
if data:
if config['file_ext'] == 'csv' or (
config['file_ext'] == 'txt' and self.__config['txt_file_data_view'] == 'TABLE'):
return self._convert_table_view_data(config, data)
elif config['file_ext'] == 'txt' and self.__config['txt_file_data_view'] == 'SLICED':
return self._convert_slices_view_data(data)
elif config['file_ext'] == 'json':
return self._convert_json_file(data)
else:
raise Exception('Incorrect file extension or file data view mode')

97
thingsboard_gateway/gateway/duplicate_detector.py
View File

@@ -39,60 +39,61 @@ class DuplicateDetector:
raise NotImplementedError("Persisting feature for latest attributes/telemetry values is not implemented!")
def filter_data(self, connector_name, new_data):
in_data_filter_enabled = new_data.get(SEND_ON_CHANGE_PARAMETER)
if not in_data_filter_enabled or not isinstance(in_data_filter_enabled, bool):
return new_data
ttl = new_data.get(SEND_ON_CHANGE_TTL_PARAMETER)
device_name = new_data[DEVICE_NAME_PARAMETER]
if not in_data_filter_enabled:
connector = self._connectors.get(connector_name)
if not connector or not connector.is_filtering_enable(device_name):
if new_data:
in_data_filter_enabled = new_data.get(SEND_ON_CHANGE_PARAMETER)
if not in_data_filter_enabled or not isinstance(in_data_filter_enabled, bool):
return new_data
elif ttl is None:
ttl = connector.get_ttl_for_duplicates(device_name)
now = int(time() * 1000)
to_send = {ATTRIBUTES_PARAMETER: [], TELEMETRY_PARAMETER: []}
ttl = new_data.get(SEND_ON_CHANGE_TTL_PARAMETER)
device_name = new_data[DEVICE_NAME_PARAMETER]
if not in_data_filter_enabled:
connector = self._connectors.get(connector_name)
if not connector or not connector.is_filtering_enable(device_name):
return new_data
elif ttl is None:
ttl = connector.get_ttl_for_duplicates(device_name)
remaining_attributes_count = 0
filtered_attributes_count = 0
for attribute in new_data[ATTRIBUTES_PARAMETER]:
for key, new_value in attribute.items():
if self._update_latest_attribute_value(device_name, key, new_value, now, ttl):
to_send[ATTRIBUTES_PARAMETER].append(attribute)
remaining_attributes_count += 1
else:
filtered_attributes_count += 1
now = int(time() * 1000)
to_send = {ATTRIBUTES_PARAMETER: [], TELEMETRY_PARAMETER: []}
remaining_telemetry_count = 0
filtered_telemetry_count = 0
for ts_kv_list in new_data[TELEMETRY_PARAMETER]:
ts_added = False
ts = ts_kv_list.get(TELEMETRY_TIMESTAMP_PARAMETER)
ts_values = {}
for key, new_value in ts_kv_list.get(TELEMETRY_VALUES_PARAMETER, ts_kv_list).items():
if self._update_latest_telemetry_value(device_name, key, new_value, ts if ts else now, ttl):
ts_values[key] = new_value
ts_added = True
remaining_telemetry_count += 1
else:
filtered_telemetry_count += 1
if ts_added:
to_send[TELEMETRY_PARAMETER].append(
{TELEMETRY_TIMESTAMP_PARAMETER: ts, TELEMETRY_VALUES_PARAMETER: ts_values} if ts else ts_values)
remaining_attributes_count = 0
filtered_attributes_count = 0
for attribute in new_data[ATTRIBUTES_PARAMETER]:
for key, new_value in attribute.items():
if self._update_latest_attribute_value(device_name, key, new_value, now, ttl):
to_send[ATTRIBUTES_PARAMETER].append(attribute)
remaining_attributes_count += 1
else:
filtered_attributes_count += 1
if remaining_attributes_count > 0 or remaining_telemetry_count > 0:
log.debug("[%s] '%s' changed attributes %d from %d, changed telemetry %d from %d",
connector_name, device_name,
remaining_attributes_count, remaining_attributes_count + filtered_attributes_count,
remaining_telemetry_count, remaining_telemetry_count + filtered_telemetry_count)
to_send[DEVICE_NAME_PARAMETER] = device_name
to_send[DEVICE_TYPE_PARAMETER] = new_data[DEVICE_TYPE_PARAMETER]
return to_send
remaining_telemetry_count = 0
filtered_telemetry_count = 0
for ts_kv_list in new_data[TELEMETRY_PARAMETER]:
ts_added = False
ts = ts_kv_list.get(TELEMETRY_TIMESTAMP_PARAMETER)
ts_values = {}
for key, new_value in ts_kv_list.get(TELEMETRY_VALUES_PARAMETER, ts_kv_list).items():
if self._update_latest_telemetry_value(device_name, key, new_value, ts if ts else now, ttl):
ts_values[key] = new_value
ts_added = True
remaining_telemetry_count += 1
else:
filtered_telemetry_count += 1
if ts_added:
to_send[TELEMETRY_PARAMETER].append(
{TELEMETRY_TIMESTAMP_PARAMETER: ts, TELEMETRY_VALUES_PARAMETER: ts_values} if ts else ts_values)
log.info("[%s] '%s' device data has not been changed", connector_name, device_name)
return None
if remaining_attributes_count > 0 or remaining_telemetry_count > 0:
log.debug("[%s] '%s' changed attributes %d from %d, changed telemetry %d from %d",
connector_name, device_name,
remaining_attributes_count, remaining_attributes_count + filtered_attributes_count,
remaining_telemetry_count, remaining_telemetry_count + filtered_telemetry_count)
to_send[DEVICE_NAME_PARAMETER] = device_name
to_send[DEVICE_TYPE_PARAMETER] = new_data[DEVICE_TYPE_PARAMETER]
return to_send
log.info("[%s] '%s' device data has not been changed", connector_name, device_name)
return None
@staticmethod
def _create_device_latest_data():