using Aitex.Core.RT.Event;
using Aitex.Core.RT.Log;
using Aitex.Core.Util;
using MECF.Framework.Common.Equipment;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Aitex.Core.RT.DataCollection.HighPerformance
{
///
/// 数据收集器缓存对象。
///
///
/// 每个Table对应一个DataRecordCache对象。该对象负责缓存数据,而不提供任何写数据库相关操作。
/// 周期性调用方法获取数据库写入表达式进行持久化操作。
///
internal class DataTraceCache : IDataTraceCache
{
#region Variables
internal const int SQL_BUILD_DURATION_TOO_SLOW_MS = 500;
internal const int MIN_CACHE_SIZE = 3000;
internal const int MIN_CACHE_PERIOD_MS = 10;
internal const int MIN_PERSIST_ITEMS = 100;
private const int MAX_SIZE_SQL_EXPR = 1000000; // approx. 5MB
private readonly object _syncRoot = new();
private readonly int _maxItemsPerPersist;
private readonly int _minCachePeriodMs;
private readonly int _maxCacheSize;
private readonly string _tableName;
private readonly string _insertExpression;
private readonly Queue _qCachedTimestamps;
private readonly Stopwatch _stopwatch = new();
private readonly StringBuilder _sqlExpr = new (MAX_SIZE_SQL_EXPR); // preallocate 5MB
private readonly StringBuilder _sqlValuesList = new (MAX_SIZE_SQL_EXPR); // preallocate 5MB
private readonly R_TRIG _rTrigCacheHalfFull = new ();
private readonly R_TRIG _rTrigCacheAlmostFull = new ();
private readonly R_TRIG _rTrigCacheFull = new ();
private readonly R_TRIG _rTrigBuildSqlExprTooSlow = new ();
private int _lastPersistRows = 0;
private double _lastCachePeriodMs = 0;
#endregion
#region Constructors
///
/// 创建数据记录器缓存对象。
///
/// 当前Cache对应的数据库表名称。
/// 当前Cache对应的分组名称。
/// 数据获取器对象集合,保存当前Cache需要缓存的数据源。
/// 最小缓存周期,如果外部频繁调用方法请求缓存数据,该周期内的重复请求被忽略,以避免高频缓存导致性能下降。
/// 每次持久化的最大行数,避免一次性对数据库写入太多数据造成性能问题。
/// 数据缓存的最大项目数。
///
public DataTraceCache(string tableName, string category, IReadOnlyList dataBuffers, int minCachePeriodMs = 50, int maxItemsPerPersist = 2000, int maxCacheSize = MIN_CACHE_SIZE)
{
Debug.Assert(dataBuffers is { Count: > 0 }, "Incorrect Data Holders List.");
Category = category;
DataBuffers = dataBuffers;
_tableName = tableName;
_maxItemsPerPersist = maxItemsPerPersist < MIN_PERSIST_ITEMS ? MIN_PERSIST_ITEMS : maxItemsPerPersist;
_minCachePeriodMs = minCachePeriodMs < MIN_CACHE_PERIOD_MS ? MIN_CACHE_PERIOD_MS : minCachePeriodMs;
_maxCacheSize = maxCacheSize < MIN_CACHE_SIZE ? MIN_CACHE_SIZE : maxCacheSize;
_qCachedTimestamps = new Queue(_maxCacheSize);
// 必须传入有效的数据收集器列表。
if (dataBuffers == null || dataBuffers.Count == 0)
throw new ArgumentException("数据缓冲器列表不能为空。", nameof(dataBuffers));
// 检查数据收集器列表中是否存在重复的序号。
if (dataBuffers.GroupBy(dc => dc.Index).Any(g => g.Count() > 1))
throw new ArgumentException("数据缓冲器列表中存在重复的序号。", nameof(dataBuffers));
// 检查数据收集器列表中是否存在重复的名称。
if (dataBuffers.GroupBy(dc => dc.Name).Any(g => g.Count() > 1))
throw new ArgumentException("数据缓冲器列表中存在重复的名称。", nameof(dataBuffers));
// 检查数据收集器列表中是否存在空的获取器。
if (dataBuffers.Any(dc => dc.Read == null))
throw new ArgumentException("数据缓冲器列表中存在空的缓冲器。", nameof(dataBuffers));
_insertExpression = BuildInsertExpression();
}
#endregion
#region Properties
///
/// 返回当前缓存服务的模组名称。
///
public string Category { get; }
///
/// 返回数据获取器列表。
///
public IReadOnlyList DataBuffers { get; }
#endregion
#region Methods
///
/// 创建Insert语句。
///
///
private string BuildInsertExpression()
{
var sqlExpr = new StringBuilder();
lock (_syncRoot)
{
sqlExpr.Append($"INSERT INTO \"{_tableName}\"(\"time\" ");
foreach (var holder in DataBuffers)
{
sqlExpr.Append($",\"{holder.Name}\"");
}
sqlExpr.Append(")");
}
return sqlExpr.ToString();
}
///
/// 立即缓存当前数据。
///
public void Cache()
{
lock (_syncRoot)
{
#region Performance Monitor
// 限制最大Cache频率
if(_stopwatch.IsRunning && _stopwatch.ElapsedMilliseconds < _minCachePeriodMs)
return;
//TODO 需要限制Q的最大数据量,并且检测HalfFull和Full事件,并触发警告,可能PC性能出现问题
if (_qCachedTimestamps.Count >= _maxCacheSize)
{
// 缓存已满,不再缓存数据
_rTrigCacheFull.CLK = true;
if(_rTrigCacheFull.Q)
EV.PostWarningLog(ModuleName.System.ToString(), $"[{Category}]DataRecorderCache Full, Capacity {_maxCacheSize} items");
return;
}
if (_qCachedTimestamps.Count >= _maxCacheSize / 2)
{
// 缓存的数据量已经超过一半,触发警告
_rTrigCacheHalfFull.CLK = true;
if(_rTrigCacheHalfFull.Q)
EV.PostWarningLog(ModuleName.System.ToString(), $"[{Category}]DataRecorderCache Half full, Capacity {_maxCacheSize} items");
}
else if (_qCachedTimestamps.Count >= _maxCacheSize * 0.9)
{
// 缓存的数据量已经超过90%,触发警告
_rTrigCacheAlmostFull.CLK = true;
if(_rTrigCacheAlmostFull.Q)
EV.PostWarningLog(ModuleName.System.ToString(), $"[{Category}]DataRecorderCache Almost full, Usage {_qCachedTimestamps.Count} of {_maxCacheSize} items");
}
else
{
_rTrigCacheFull.CLK = false;
_rTrigCacheHalfFull.CLK = false;
_rTrigCacheAlmostFull.CLK = false;
}
#endregion
_lastCachePeriodMs = _stopwatch.ElapsedMilliseconds;
_stopwatch.Restart();
var ts = DateTime.Now.Ticks;
if (_qCachedTimestamps.Contains(ts))
{
LOG.Error($"时间戳{ts}已经存在。");
return;
}
var ret = Parallel.ForEach(DataBuffers, holder => { holder.Cache(ts); });
if (ret.IsCompleted)
{
_qCachedTimestamps.Enqueue(ts);
}
}
}
///
/// 获取将缓存数据写入数据库的SQL语句。
///
///
///
public string GetInsertSql()
{
lock (_syncRoot)
{
// 如果没数据,直接退出
if (_qCachedTimestamps.Count == 0)
return "";
var sw = new Stopwatch();
sw.Start();
var pickedCount = 0;
_sqlValuesList.Clear();
while (true)
{
// 没有缓存的数据了,退出
if (_qCachedTimestamps.Count == 0)
break;
var ts = _qCachedTimestamps.Dequeue();
{
_sqlValuesList.Append("(");
_sqlValuesList.Append($"'{ts}',");
foreach (var holder in DataBuffers)
{
var value = holder.Get(ts);
_sqlValuesList.Append(value);
_sqlValuesList.Append(",");
}
_sqlValuesList.Remove(_sqlValuesList.Length - 1, 1);
_sqlValuesList.Append("),");
}
pickedCount++;
if (pickedCount >= _maxItemsPerPersist)
break;
}
if (_sqlValuesList.Length <= 0)
{
_lastPersistRows = 0;
return "";
}
else
{
_lastPersistRows = pickedCount;
}
_sqlValuesList.Remove(_sqlValuesList.Length - 1, 1);
_sqlValuesList.Append(";");
// 拼接完整的SQL语句
_sqlExpr.Clear();
_sqlExpr.Append(_insertExpression);
_sqlExpr.Append(" VALUES ");
_sqlExpr.Append(_sqlValuesList);
sw.Stop();
//TODO 加入性能监视器,如果执行时间过长,则引发Warn事件;
_rTrigBuildSqlExprTooSlow.CLK = sw.ElapsedMilliseconds > SQL_BUILD_DURATION_TOO_SLOW_MS;
if (_rTrigBuildSqlExprTooSlow.Q)
LOG.Warning($"{nameof(DataTraceCache)} Build sql expression too slow, took {sw.ElapsedMilliseconds:F1}ms which was greater than {SQL_BUILD_DURATION_TOO_SLOW_MS:F1}ms");
// EV.PostWarningLog(ModuleName.System.ToString(), $"DataRecorderCache Build SQL Expression Too Slow, Took {sw.ElapsedMilliseconds:F1}ms greater than {SQL_BUILD_DURATION_TOO_SLOW_MS:F1}ms");
Debug.WriteLine(
$"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] GenSQL:Ln{_lastPersistRows}/{sw.ElapsedMilliseconds}ms/{_sqlExpr.Length} bytes, LastCachePeriod:{_lastCachePeriodMs}ms, CacheRemained: {_qCachedTimestamps.Count}",
$"{nameof(DataTraceCache)} - {Category}");
return _sqlExpr.ToString();
}
}
#endregion
}
}