I've tried that code with a single local Ignite server, started with
default configuration, the result is:
Time for PutAll: 187 ms for rows: 7500
Time for fetching all data: 128 ms for rows: 7500
However, when I removed JSON serialization and stored the objects as is
(IgniteCache<int, EmpDataSet>):
Time for PutAll: 1924 ms for rows: 7500
Time for fetching all data: 20288 ms for rows: 7500
There is a performance issue with DateTime value serialization. I'm
investigating and will get back to you with details soon.
Meanwhile, there is a *workaround*: force Timestamp serialization format
and convert all values to UTC (which is a good idea anyway):
new IgniteClientConfiguration
{
...
BinaryConfiguration = new BinaryConfiguration
{
Serializer = new BinaryReflectiveSerializer
{
ForceTimestamp = true
}
}
};
Then call ToUniversalTime() on all DateTime values. The result is:
Time for PutAll: 761 ms for rows: 7500
Time for fetching all data: 645 ms for rows: 7500
For reference, JSON serialization of those 7500 rows (CreateMapFromList)
takes ~600ms on my machine.
On Tue, Jan 14, 2020 at 4:09 PM Tunas <[email protected]>
wrote:
> I am using Ignite 2.7.6 dll for Ignite and newtonSoft json for converting
> C#
> objects to json.
> Ignite configuration is same as mentioned allow. (Server and client are
> running on different machine)
>
> Sorry code class is little bit lengthy.
>
> One thing observed before publishing this code: addition of properties of
> type ObservableCollection<T> is taking more time.
>
> class Program
> {
> public static IIgniteClient IgniteThinClient;
> private static IgniteClientConfiguration
> GetIgniteClientConfiguration()
> {
> return new IgniteClientConfiguration
> {
> Endpoints = new[] { "MyWorkStation:10800" },
> SocketTimeout = TimeSpan.FromSeconds(60)
> };
> }
> static void Main(string[] args)
> {
> int rows = 7500;
> var lst = GetData(rows);
>
> Ignition.ClientMode = true;
> IgniteThinClient =
> Ignition.StartClient(GetIgniteClientConfiguration());
> var cache = IgniteThinClient.GetOrCreateCache<int,
> string>("TestSlowness");
>
> var map1 = CreateMapFromList(lst);
> Stopwatch sw = new Stopwatch();
> sw.Start();
> cache.PutAll(map1);
> sw.Stop();
> Console.WriteLine($"Time for PutAll: {sw.ElapsedMilliseconds}
> ms
> for rows: {rows}");
> sw.Reset();
>
>
> ICacheClient<int, string> cacheClient =
> IgniteThinClient.GetOrCreateCache<int, string>("TestSlowness");
> if(cacheClient!=null)
> {
> sw.Start();
> *var cacheEntries = cacheClient.Query(new
> Apache.Ignite.Core.Cache.Query.ScanQuery<int, string>(null)).GetAll();*
> sw.Stop();
> Console.WriteLine($"Time for fetching all data:
> {sw.ElapsedMilliseconds} ms for rows: {rows}");
> }
> }
>
> //Actually below one is generic method(for simplicity changed to
> non-generic)
> private static ConcurrentDictionary<int, string>
> CreateMapFromList(IEnumerable<EmpDataSet> collection)
> {
> var map = new ConcurrentDictionary<int, string>();
> if (collection != null && collection.Any())
> {
> foreach (EmpDataSet item in collection)
> {
> var val = SerializedEntity(item);
> if (!string.IsNullOrWhiteSpace(val))
> {
> map.TryAdd(item.DtId, val);
> }
> }
> }
> return map;
> }
>
>
> private static List<EmpDataSet> GetData(int rows = 7000)
> {
> Random rand = new Random();
>
> var lst = new List<EmpDataSet>();
> for (int i=1; i<=rows;i++)
> {
> var obj1 = new EmpDataSet();
> obj1.DtId = i;
> obj1.EmpRef = rand.Next(rows + rows);
> obj1.DtName = RandomString(rand.Next(15, 20));
> obj1.DraftDtId = rand.Next(rows + rows);
> obj1.ApprovedDtId = rand.Next(rows + rows);
> obj1.Description = RandomString(rand.Next(20, 60));
> obj1.Code1 = RandomString(rand.Next(1, 10),
> Convert.ToBoolean(rand.Next(0, 1)));
> obj1.Code2 = RandomString(rand.Next(1, 10),
> Convert.ToBoolean(rand.Next(0, 1)));
> obj1.Label = RandomString(rand.Next(1, 10),
> Convert.ToBoolean(rand.Next(0, 1)));
> obj1.Etype = (EType)(rand.Next(1, 8));
> obj1.Estatus = (EStatus)(rand.Next(1, 4));
> obj1.Notes = RandomString(30,
> Convert.ToBoolean(rand.Next(0,
> 1)));
> obj1.Flag1 = Convert.ToBoolean(rand.Next(0, 1));
> obj1.Flag2 = Convert.ToBoolean(rand.Next(0, 1));
> obj1.Flag3 = Convert.ToBoolean(rand.Next(0, 1));
> obj1.EmpAreaMap = ConstructEmpAreaMap(rand);
> obj1.EcType = (EcType)(rand.Next(1, 2));
> obj1.Ref1 = rand.Next(50, 100);
> obj1.Ref1Name = RandomString(10,
> Convert.ToBoolean(rand.Next(0, 1)));
> obj1.DgId = rand.Next(24, 48);
> obj1.EmpRegs = GetEmpRegs(rand);
> obj1.DRegs = null;
> obj1.Fmap = new FMap { Id = rand.Next(1, rows + rows), Flts
> = ConstructFlts(rand, rand.Next(1, 4)) };
> obj1.group1 = ConstructAgroup(rand);
> obj1.group2 = ConstructAgroup(rand);
> obj1.Str1 = RandomString(rand.Next(1, 50),
> Convert.ToBoolean(rand.Next(0, 1)));
> obj1.Str2 = RandomString(rand.Next(1, 50),
> Convert.ToBoolean(rand.Next(0, 1)));
> obj1.Str3 = RandomString(rand.Next(1, 50),
> Convert.ToBoolean(rand.Next(0, 1)));
> obj1.Str4 = RandomString(rand.Next(1, 50),
> Convert.ToBoolean(rand.Next(0, 1)));
> obj1.Str5 = RandomString(rand.Next(1, 50),
> Convert.ToBoolean(rand.Next(0, 1)));
> lst.Add(obj1);
> }
> return lst;
> }
>
> private static Agroup ConstructAgroup(Random rand)
> {
> int rows = rand.Next(10000, 20000);
> var obj1 = new Agroup
> {
> Id = rand.Next(1, rows + rows),
> Ref = RandomString(rand.Next(1, 40),
> Convert.ToBoolean(rand.Next(0, 1))),
> Flts = ConstructFlts(rand, rand.Next(1, 4)),
> EmpRegs = new
> ObservableCollection<EmpReg>(GetEmpRegs(rand))
> };
> return obj1;
> }
>
> private static Dictionary<EmpArea, List<EmpArea>>
> ConstructEmpAreaMap(Random rand,
> int rows=2, int innerCollection=3)
> {
> var map = new Dictionary<EmpArea, List<EmpArea>>();
> for(int t=1; t<=rows;t++)
> {
> var obj2 = ConstructEmpAreaObj(rand);
> var lst = new List<EmpArea>();
> for (int i = 1; i <= innerCollection; i++)
> {
> var obj1 = ConstructEmpAreaObj(rand);
> lst.Add(obj1);
> }
> map.Add(obj2, lst);
> }
> return map;
> }
>
> private static EmpArea ConstructEmpAreaObj(Random rand)
> {
> var obj1 = new EmpArea();
> obj1.Id = rand.Next(1, 10);
> obj1.Name = RandomString(25);
> obj1.Description = RandomString(50);
> obj1.Freq = rand.Next(1, 100);
> obj1.StartNum = rand.Next(1, 50);
> return obj1;
> }
>
> private static ObservableCollection<Flt> ConstructFlts(Random rand,
> int rows=3)
> {
> var lst = new ObservableCollection<Flt>();
> for(int i=0;i<rows;i++)
> {
> var obj = new Flt();
> obj.Id1 = rand.Next(1, 50);
> obj.Id2 = rand.Next(1, 50);
> obj.Id3 = rand.Next(1, 50);
> obj.StartDate = DateTime.Today.AddDays(rand.Next(100,
> 200));
> obj.EndDate = DateTime.Today.AddDays(rand.Next(1, 100));
> obj.Share = Convert.ToDecimal(rand.NextDouble());
> obj.Str1 = RandomString(rand.Next(10, 30),
> Convert.ToBoolean(rand.Next(0,1)));
> obj.Str2 = RandomString(rand.Next(10, 30),
> Convert.ToBoolean(rand.Next(0, 1)));
> obj.Str3 = RandomString(rand.Next(10, 30),
> Convert.ToBoolean(rand.Next(0, 1)));
> lst.Add(obj);
> }
> return lst;
> }
>
> private static List<EmpReg> GetEmpRegs(Random rand, int rows=2)
> {
> rows = rand.Next(1, 4);
> var lst = new List<EmpReg>();
> for(int i=0;i<rows;i++)
> {
> var obj = new EmpReg();
> obj.StartDate = DateTime.Today.AddDays(-1 * rand.Next(500,
> 1000));
> obj.EndDate = DateTime.Today.AddDays(-1 * rand.Next(1,
> 500));
> obj.InceptionDate = DateTime.Today.AddDays(-1 *
> rand.Next(1000, 1100));
> obj.VDate = DateTime.Today.AddDays(-1 * rand.Next(1, 10));
> obj.Id = rand.Next(1, 100);
> obj.Name = RandomString(20, Convert.ToBoolean(rand.Next(0,
> 1)));
> obj.Order = rand.Next(1000, 2000);
> lst.Add(obj);
> }
> return lst;
> }
> private static string RandomString(int size, bool lowerCase=false)
> {
> StringBuilder builder = new StringBuilder();
> Random random = new Random();
> char ch;
> for (int i = 0; i < size; i++)
> {
> ch = Convert.ToChar(Convert.ToInt32(Math.Floor(26 *
> random.NextDouble() + 65)));
> builder.Append(ch);
> }
> if (lowerCase)
> return builder.ToString().ToLower();
> return builder.ToString();
> }
> private static string SerializedEntity<T>(T value)
> {
> try
> {
> var settings = new JsonSerializerSettings
> {
> PreserveReferencesHandling =
> PreserveReferencesHandling.Objects,
> NullValueHandling = NullValueHandling.Ignore,
> ReferenceLoopHandling =
> ReferenceLoopHandling.Serialize,
> };
> settings.ReferenceLoopHandling =
> ReferenceLoopHandling.Serialize;
> var json = JsonConvert.SerializeObject(value, settings);
> return json;
> }
> catch (Exception ex)
> {
> Console.WriteLine(ex);
> }
> return string.Empty;
> }
> }
>
>
>
> public class EmpDataSet
> {
> public int DtId { get; set; }
> public int EmpRef { get; set; }
> public string DtName { get; set; }
> public int? DraftDtId { get; set; }
> public int? ApprovedDtId { get; set; }
> public string Description { get; set; }
> public string Code1 { get; set; }
> public string Code2 { get; set; }
> public string Label { get; set; }
> public EType Etype { get; set; }
> public EStatus Estatus { get; set; }
> public string Notes { get; set; }
> public bool Flag1 { get; set; }
> public bool Flag2 { get; set; }
> public bool Flag3 { get; set; }
> public Dictionary<EmpArea, List<EmpArea>> EmpAreaMap { get; set;
> }
> public EcType? EcType { get; set; }
> public int? Ref1 { get; set; }
> public string Ref1Name { get; set; }
> public int? DgId { get; set; }
> private List<EmpReg> _empRegs;
> public List<EmpReg> EmpRegs
> {
> get => _empRegs ?? (_empRegs = new List<EmpReg>());
> set => _empRegs = value;
> }
> private List<EmpReg> _dRegs;
> public List<EmpReg> DRegs
> {
> get => _dRegs ?? (_dRegs = new List<EmpReg>());
> set => _dRegs = value;
> }
> public FMap Fmap { get; set; }
> public Agroup group1 { get; set; }
> public Agroup group2 { get; set; }
> public string Str1 { get; set; }
> public string Str2 { get; set; }
> public string Str3 { get; set; }
> public string Str4 { get; set; }
> public string Str5 { get; set; }
>
> }
> public class EmpArea
> {
> public int Id { get; set; }
> public string Name { get; set; }
> public string Description { get; set; }
> public int Freq { get; set; }
> public int StartNum { get; set; }
> }
> public enum EType
> {
> A=1,
> B=2,
> C=3,
> D=4,
> E=5,
> F=6,
> G=7,
> H=8
> }
> public enum EStatus
> {
> A1=1,
> A2=2,
> A3=3,
> A4=4
> }
> public enum EcType
> {
> S=1,
> M=2
> }
> public class EmpReg
> {
> public DateTime? StartDate { get; set; }
> public DateTime? EndDate { get; set; }
> public DateTime? InceptionDate { get; set; }
> public DateTime? VDate { get; set; }
> public int Id { get; set; }
> public string Name { get; set; }
> public int Order { get; set; }
>
> }
> public class FMap
> {
> public int Id { get; set; }
> public ObservableCollection<Flt> Flts{ get; set; }
> }
> public class Agroup
> {
> public int Id { get; set; }
> public string Ref { get; set; }
> public ObservableCollection<Flt> Flts { get; set; }
> * public ObservableCollection<EmpReg> EmpRegs { get; set; }*
> }
> public class Flt
> {
> public int Id1 { get; set; }
> public int Id2 { get; set; }
> public int Id3 { get; set; }
> public DateTime? StartDate { get; set; }
> public DateTime? EndDate { get; set; }
> public decimal Share { get; set; }
> public string Str1 { get; set; }
> public string Str2 { get; set; }
> public string Str3 { get; set; }
> }
>
>
>
> --
> Sent from: http://apache-ignite-users.70518.x6.nabble.com/
>
