优化无损压缩算法，提高压缩速度

Common/Cdy.Tag.Common/Common/CustomQueue.cs

+//==============================================================
+//  Copyright (C) 2020  Inc. All rights reserved.
+//
+//==============================================================
+//  Create by 种道洋 at 2020/5/23 23:58:40.
+//  Version 1.0
+//  种道洋
+//==============================================================
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Cdy.Tag
+{
+    public class CustomQueue<T>
+    {
+
+        #region ... Variables  ...
+        private T[] mColections;
+        private int mCount = 0;
+        #endregion ...Variables...
+
+        #region ... Events     ...
+
+        #endregion ...Events...
+
+        #region ... Constructor...
+        
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="count"></param>
+        public CustomQueue(int count)
+        {
+            mColections = new T[count];
+            mCount = count;
+        }
+
+        #endregion ...Constructor...
+
+        #region ... Properties ...
+
+
+
+        public int Index { get; set; }
+        #endregion ...Properties...
+
+        #region ... Methods    ...
+        
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="value"></param>
+        public void Insert(T value)
+        {
+            Index++;
+            mColections[Index] = value;
+        }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <returns></returns>
+        public T Remove()
+        {
+            Index--;
+            if (Index < 0)
+            {
+                Index = -1;
+                return default(T);
+            }
+            else
+            {
+                return mColections[Index];
+            }
+        }
+
+        public T Get(int index)
+        {
+            return mColections[index];
+        }
+
+        #endregion ...Methods...
+
+        #region ... Interfaces ...
+
+        #endregion ...Interfaces...
+    }
+}

Common/Cdy.Tag.Common/Common/MemoryBlock.cs

@@ -47,7 +47,7 @@ namespace Cdy.Tag
         private object mUserSizeLock = new object();
 
 
-        public int BufferItemSize = 1024 * 1024 * 128;
+        public int BufferItemSize = 1024 * 1024 * 4;
 
         public static byte[] zoreData = new byte[1024 * 10];
 

Common/Cdy.Tag.Consume/Cdy.Tag.Consume.csproj

@@ -8,6 +8,10 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\Cdy.Tag\Cdy.Tag.csproj" />
   </ItemGroup>

Common/Cdy.Tag/His/CompressUnitManager.cs

@@ -26,11 +26,18 @@ namespace Cdy.Tag
         /// </summary>
         private Dictionary<int, CompressUnitbase> mCompressUnit = new Dictionary<int, CompressUnitbase>();
 
+        /// <summary>
+        /// 
+        /// </summary>
+        private Dictionary<int, Queue<CompressUnitbase>> mPoolCompressUnits = new Dictionary<int, Queue<CompressUnitbase>>();
+
         /// <summary>
         /// 
         /// </summary>
         public static CompressUnitManager Manager = new CompressUnitManager();
 
+
+
         #endregion ...Variables...
 
         #region ... Events     ...
@@ -54,7 +61,52 @@ namespace Cdy.Tag
         /// <returns></returns>
         public CompressUnitbase GetCompress(int type)
         {
-            return mCompressUnit.ContainsKey(type) ? mCompressUnit[type] : null;
+            lock (mPoolCompressUnits)
+            {
+                if (mCompressUnit.ContainsKey(type))
+                {
+                    if (mPoolCompressUnits.ContainsKey(type) && mPoolCompressUnits[type].Count > 0)
+                    {
+                        return mPoolCompressUnits[type].Dequeue();
+                    }
+                    else
+                    {
+                        return mCompressUnit[type].Clone();
+                    }
+                }
+            }
+            return null;
+        }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="type"></param>
+        /// <returns></returns>
+        public CompressUnitbase GetCompressQuick(int type)
+        {
+            return mCompressUnit.ContainsKey(type)?mCompressUnit[type]:null;
+        }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="compress"></param>
+        public void ReleaseCompress(CompressUnitbase compress)
+        {
+            lock (mPoolCompressUnits)
+            {
+                if (mPoolCompressUnits.ContainsKey(compress.TypeCode))
+                {
+                    mPoolCompressUnits[compress.TypeCode].Enqueue(compress);
+                }
+                else
+                {
+                    var dd = new Queue<CompressUnitbase>();
+                    dd.Enqueue(compress);
+                    mPoolCompressUnits.Add(compress.TypeCode, dd);
+                }
+            }
         }
 
         /// <summary>

Common/Cdy.Tag/His/CompressUnitbase.cs

@@ -91,6 +91,7 @@ namespace Cdy.Tag
 
         #region ... Methods    ...
 
+
         /// <summary>
         /// 
         /// </summary>

Common/Cdy.Tag/Interface/HisQueryResult.cs

@@ -838,6 +838,95 @@ namespace Cdy.Tag
             AddPoint(value.X, value.Y, value.Z, time, qulity);
         }
 
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="index"></param>
+        /// <returns></returns>
+        public T GetValue(int index)
+        {
+            object re = null;
+            switch (mDataType)
+            {
+                case 0:
+                    re = Convert.ToBoolean(MemoryHelper.ReadByte((void*)handle, index));
+                    break;
+                case 1:
+                    re = MemoryHelper.ReadByte((void*)handle, index);
+                    break;
+                case 2:
+                    re = MemoryHelper.ReadShort((void*)handle, index * 2);
+                    break;
+                case 3:
+                    re = MemoryHelper.ReadShort((void*)handle, index * 2);
+                    break;
+                case 4:
+                    re = MemoryHelper.ReadInt32((void*)handle, index * 4);
+                    break;
+                case 5:
+                    re = MemoryHelper.ReadUInt32((void*)handle, index * 4);
+                    break;
+                case 6:
+                    re = MemoryHelper.ReadInt64((void*)handle, index * 8);
+                    break;
+                case 7:
+                    re = MemoryHelper.ReadUInt64((void*)handle, index * 8);
+                    break;
+                case 8:
+                    re = MemoryHelper.ReadFloat((void*)handle, index * 4);
+                    break;
+                case 9:
+                    re = MemoryHelper.ReadDouble((void*)handle, index * 8);
+                    break;
+                case 10:
+                    re = MemoryHelper.ReadDateTime((void*)handle, index * 8);
+                    break;
+                case 11:
+
+                    int cc = 0;
+                    int pos = 0;
+                    while (true)
+                    {
+                        if (cc >= index)
+                        {
+                            break;
+                        }
+                        //pos += (mDataBuffer[pos]+1);
+                        pos += MemoryHelper.ReadByte((void*)handle, pos) + 1;
+                        cc++;
+                    }
+                    re = new string((char*)handle, pos + 1, MemoryHelper.ReadByte((void*)handle, pos));
+                    break;
+                case 12:
+                    var x = MemoryHelper.ReadInt32((void*)handle, index * 8);
+                    var y = MemoryHelper.ReadInt32((void*)handle, index * 8 + 4);
+                    re = new IntPointData(x, y);
+                    break;
+                case 13:
+                    re = new UIntPointData(MemoryHelper.ReadUInt32((void*)handle, index * 8), MemoryHelper.ReadUInt32((void*)handle, index * 8 + 4));
+                    break;
+                case 14:
+                    re = new IntPoint3Data(MemoryHelper.ReadInt32((void*)handle, index * 12), MemoryHelper.ReadInt32((void*)handle, index * 12 + 4), MemoryHelper.ReadInt32((void*)handle, index * 12 + 8));
+                    break;
+                case 15:
+                    re = new UIntPoint3Data(MemoryHelper.ReadUInt32((void*)handle, index * 12), MemoryHelper.ReadUInt32((void*)handle, index * 12 + 4), MemoryHelper.ReadUInt32((void*)handle, index * 12 + 8));
+                    break;
+                case 16:
+                    re = new LongPointData(MemoryHelper.ReadInt64((void*)handle, index * 16), MemoryHelper.ReadInt64((void*)handle, index * 16 + 8));
+                    break;
+                case 17:
+                    re = new ULongPointData(MemoryHelper.ReadUInt64((void*)handle, index * 16), MemoryHelper.ReadUInt64((void*)handle, index * 16 + 8));
+                    break;
+                case 18:
+                    re = new LongPoint3Data(MemoryHelper.ReadInt64((void*)handle, index * 24), MemoryHelper.ReadInt64((void*)handle, index * 24 + 8), MemoryHelper.ReadInt64((void*)handle, index * 24 + 16));
+                    break;
+                case 19:
+                    re = new ULongPoint3Data(MemoryHelper.ReadUInt64((void*)handle, index * 24), MemoryHelper.ReadUInt64((void*)handle, index * 24 + 8), MemoryHelper.ReadUInt64((void*)handle, index * 24 + 16));
+                    break;
+            }
+            return (T)re;
+        }
+
         /// <summary>
         /// 
         /// </summary>

Develop/DBStudio/Program.cs

@@ -396,6 +396,7 @@ namespace DBStudio
         {
             StringBuilder re = new StringBuilder();
             re.AppendLine();
+            re.AppendLine("save                                                 // save database ");
             re.AppendLine("start                                                 // start database ");
             re.AppendLine("restart                                                 // restart database ");
             re.AppendLine("stop                                                 // stop database ");

Proxy/DBHighApi/DBHighApi.csproj

@@ -9,6 +9,10 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\..\Common\Cdy.Tag.Common\Cdy.Tag.Common.csproj" />
     <ProjectReference Include="..\..\Common\Cdy.Tag\Cdy.Tag.csproj" />

RunTime/DBHisDataServer/DBHisDataServer.csproj

@@ -10,6 +10,11 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
+    <Prefer32Bit>false</Prefer32Bit>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\..\Common\Cdy.Tag.Common\Cdy.Tag.Common.csproj" />
     <ProjectReference Include="..\..\Common\Cdy.Tag.Network\Cdy.Tag.Network.csproj" />

RunTime/DBRuntime.Proxy/DBRuntime.Proxy.csproj

@@ -8,6 +8,10 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\..\Common\Cdy.Tag.Driver\Cdy.Tag.Driver.csproj" />
     <ProjectReference Include="..\..\Common\Cdy.Tag.Network\Cdy.Tag.Network.csproj" />

RunTime/DBRuntime.Real/DBRuntime.Real.csproj

@@ -8,6 +8,10 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
   <Target Name="PostBuild" AfterTargets="PostBuildEvent">
     <Exec Command="copy &quot;$(TargetPath)&quot; &quot;$(SolutionDir)\Output&quot; /y&#xD;&#xA;if exist  &quot;$(TargetDir)$(TargetName).XML&quot; copy &quot;$(TargetDir)$(TargetName).XML&quot; &quot;$(SolutionDir)\Output\Xml&quot; /y&#xD;&#xA;if exist  &quot;$(TargetDir)$(TargetName).pdb&quot;  copy &quot;$(TargetDir)$(TargetName).pdb&quot; &quot;$(SolutionDir)\Output&quot; /y&#xD;&#xA;copy &quot;$(TargetDir)Config\*.cfg&quot; &quot;$(SolutionDir)\Output\Config&quot; /y&#xD;&#xA;if exist  &quot;$(TargetDir)zh-CN&quot;  copy &quot;$(TargetDir)zh-CN\*.dll&quot; &quot;$(SolutionDir)\Output\zh-CN&quot; /y&#xD;&#xA;" />
   </Target>

RunTime/DBRuntime/His/Compress/CompressEnginer.cs

@@ -102,7 +102,7 @@ namespace Cdy.Tag
                 }
             }
 
-            long psize = mTotalSize/mTargetMemorys.Count;
+            long psize = (long)(mTotalSize*1.5)/mTargetMemorys.Count;
 
             System.Threading.Tasks.Parallel.ForEach(mTargetMemorys, (vv) => {
                 vv.Value.ReAlloc(vv.Value.HeadSize + psize);

RunTime/DBRuntime/His/Compress/CompressMemory.cs

@@ -28,6 +28,8 @@ namespace Cdy.Tag
         private Dictionary<int, Tuple<long, int, int, int>> mTagAddress;
         private DateTime mCurrentTime;
         private IHisEngine mHisTagService;
+        private Dictionary<int, CompressUnitbase> mCompressCach = new Dictionary<int, CompressUnitbase>();
+        private Dictionary<int, long> dtmp = new Dictionary<int, long>();
         #endregion ...Variables...
 
         #region ... Events     ...
@@ -67,6 +69,8 @@ namespace Cdy.Tag
         /// </summary>
         public static int TagCountPerMemory { get; set; }
 
+       
+
         /// <summary>
         /// 
         /// </summary>
@@ -120,14 +124,15 @@ namespace Cdy.Tag
 
         #region ... Methods    ...
 
-        /// <summary>
-        /// 
-        /// </summary>
-        public void ResetTagAddress()
-        {
-            mTagAddress.Clear();
-            mTagAddress = null;
-        }
+        ///// <summary>
+        ///// 
+        ///// </summary>
+        //public void ResetTagAddress()
+        //{
+        //    mTagAddress.Clear();
+        //    mTagAddress = null;
+        //    mCompressCach.Clear();
+        //}
 
         /// <summary>
         /// 
@@ -135,12 +140,20 @@ namespace Cdy.Tag
         /// <param name="sourceM"></param>
         private void CheckTagAddress(MergeMemoryBlock sourceM)
         {
-            if(mTagAddress==null && sourceM!=null)
+            
+            if (mTagAddress==null && sourceM!=null)
             {
                 mTagAddress = new Dictionary<int, Tuple<long, int, int, int>>();
+                dtmp.Clear();
                 foreach (var vv in sourceM.TagAddress.Where(e=>e.Key>=Id* TagCountPerMemory && e.Key<(Id+1)* TagCountPerMemory))
                 {
                     mTagAddress.Add(vv.Key,vv.Value);
+                    dtmp.Add(vv.Key, 0);
+                    var cpt = mHisTagService.GetHisTag(vv.Key).CompressType;
+                    if(!mCompressCach.ContainsKey(cpt))
+                    {
+                        mCompressCach.Add(cpt, CompressUnitManager.Manager.GetCompressQuick(cpt).Clone());
+                    }
                 }
             }
         }
@@ -165,16 +178,15 @@ namespace Cdy.Tag
             int headOffset = 4 + 4;
             long Offset = headOffset + this.mTagAddress.Count * 8;
 
-            
-            Dictionary<int, long> dtmp = new Dictionary<int, long>();
-
             this.MakeMemoryBusy();
 
+            long ltmp1 = sw.ElapsedMilliseconds;
+
             //更新数据区域
             foreach(var vv in mTagAddress)
             {
                 var size = CompressBlockMemory(source, vv.Value.Item1, Offset,vv.Value.Item3, vv.Value.Item4, vv.Key);
-                dtmp.Add(vv.Key, Offset);
+                dtmp[vv.Key]= Offset;
                 Offset += size;
                 datasize += size;
             }
@@ -183,6 +195,8 @@ namespace Cdy.Tag
             this.WriteInt(0,(int)datasize);
             this.Write((int)this.mTagAddress.Count);
 
+            long ltmp2 = sw.ElapsedMilliseconds;
+
             int count = 0;
             foreach (var vv in dtmp)
             {
@@ -191,10 +205,12 @@ namespace Cdy.Tag
                 count+=8;
             }
 
+            long ltmp3 = sw.ElapsedMilliseconds;
+
             ServiceLocator.Locator.Resolve<IDataSerialize>().RequestToSeriseFile(this, mCurrentTime);
             sw.Stop();
-
-            LoggerService.Service.Info("CompressEnginer", Id+ "压缩完成 耗时:"+sw.ElapsedMilliseconds);
+            
+            LoggerService.Service.Info("CompressEnginer", Id+ "压缩完成 耗时:"+sw.ElapsedMilliseconds +" ltmp1:"+ltmp1 +" ltmp2:"+(ltmp2-ltmp1)+" ltmp3:"+(ltmp3-ltmp2),ConsoleColor.Red);
         }
 
         /// <summary>
@@ -219,12 +235,12 @@ namespace Cdy.Tag
 
             var comtype = histag.CompressType;//压缩类型
 
-            this.CheckAndResize(targetPosition + len);
+          //  this.CheckAndResize(targetPosition + len);
 
             //写入压缩类型
             this.WriteByte(targetPosition + 4, (byte)comtype);
 
-            var tp = CompressUnitManager.Manager.GetCompress(comtype);
+            var tp = mCompressCach[comtype];
             if (tp != null)
             {
                 tp.QulityOffset = (int)qulityoffset;
@@ -235,6 +251,7 @@ namespace Cdy.Tag
                 var size = tp.Compress(mSourceMemory, addr, this, targetPosition + 5, len) + 1;
                 this.WriteInt(targetPosition, (int)size);
                 //this.Dump();
+            
                 return size + 5;
             }
             

RunTime/DBRuntime/His/Compress/Special/DeadAreaCompressUnit.cs

@@ -65,7 +65,7 @@ namespace Cdy.Tag
         /// <param name="count"></param>
         /// <param name="emptyIds"></param>
         /// <returns></returns>
-        protected override Memory<byte> CompressValues<T>(MarshalMemoryBlock source, long offset, int count, Queue<int> emptyIds)
+        protected override Memory<byte> CompressValues<T>(MarshalMemoryBlock source, long offset, int count, CustomQueue<int> emptys, TagType type)
         {
             var deadArea = this.Parameters.ContainsKey("DeadValue") ? this.Parameters["DeadValue"] : 0;
             var deadType = (int)(this.Parameters.ContainsKey("DeadType") ? this.Parameters["DeadType"] : 0);
@@ -76,7 +76,7 @@ namespace Cdy.Tag
             bool isFirst = true;
 
             int ig = -1;
-            emptyIds.TryDequeue(out ig);
+            ig = emptys.Index >= 0 ? emptys.Remove() : -1;
 
             if (typeof(T) == typeof(byte))
             {
@@ -103,8 +103,8 @@ namespace Cdy.Tag
                     }
                     else
                     {
-                        if (emptyIds.Count > 0)
-                            emptyIds.TryDequeue(out ig);
+                       
+                            ig = emptys.Index >= 0 ? emptys.Remove() : -1;
                     }
                 }
                 return mMarshalMemory.StartMemory.AsMemory<byte>(0, (int)mMarshalMemory.Position);
@@ -134,8 +134,8 @@ namespace Cdy.Tag
                     }
                     else
                     {
-                        if (emptyIds.Count > 0)
-                            emptyIds.TryDequeue(out ig);
+                       
+                            ig = emptys.Index >= 0 ? emptys.Remove() : -1;
                     }
                 }
 
@@ -165,8 +165,8 @@ namespace Cdy.Tag
                     }
                     else
                     {
-                        if (emptyIds.Count > 0)
-                            emptyIds.TryDequeue(out ig);
+                        
+                            ig = emptys.Index >= 0 ? emptys.Remove() : -1;
                     }
                 }
 
@@ -196,8 +196,8 @@ namespace Cdy.Tag
                     }
                     else
                     {
-                        if (emptyIds.Count > 0)
-                            emptyIds.TryDequeue(out ig);
+                        
+                            ig = emptys.Index >= 0 ? emptys.Remove() : -1;
                     }
                 }
 
@@ -227,8 +227,8 @@ namespace Cdy.Tag
                     }
                     else
                     {
-                        if (emptyIds.Count > 0)
-                            emptyIds.TryDequeue(out ig);
+                        
+                            ig = emptys.Index >= 0 ? emptys.Remove() : -1;
                     }
                 }
             }
@@ -257,8 +257,8 @@ namespace Cdy.Tag
                     }
                     else
                     {
-                        if (emptyIds.Count > 0)
-                            emptyIds.TryDequeue(out ig);
+                        
+                            ig = emptys.Index >= 0 ? emptys.Remove() : -1;
                     }
                 }
 
@@ -288,8 +288,8 @@ namespace Cdy.Tag
                     }
                     else
                     {
-                        if (emptyIds.Count > 0)
-                            emptyIds.TryDequeue(out ig);
+                        
+                            ig = emptys.Index >= 0 ? emptys.Remove() : -1;
                     }
                 }
 
@@ -319,8 +319,8 @@ namespace Cdy.Tag
                     }
                     else
                     {
-                        if (emptyIds.Count > 0)
-                            emptyIds.TryDequeue(out ig);
+                       
+                            ig = emptys.Index >= 0 ? emptys.Remove() : -1;
                     }
                 }
                 return mMarshalMemory.StartMemory.AsMemory<byte>(0, (int)mMarshalMemory.Position);
@@ -350,8 +350,8 @@ namespace Cdy.Tag
                     }
                     else
                     {
-                        if (emptyIds.Count > 0)
-                            emptyIds.TryDequeue(out ig);
+                        
+                            ig = emptys.Index >= 0 ? emptys.Remove() : -1;
                     }
                 }
                 return mMarshalMemory.StartMemory.AsMemory<byte>(0, (int)mMarshalMemory.Position);