Ticket filed and fix is on the way: https://issues.apache.org/jira/browse/IGNITE-12555
On Tue, Jan 14, 2020 at 6:44 PM Pavel Tupitsyn <[email protected]> wrote: > I've tried that code with a single local Ignite server, started with > default configuration, the result is: > Time for PutAll: 187 ms for rows: 7500 > Time for fetching all data: 128 ms for rows: 7500 > > However, when I removed JSON serialization and stored the objects as is > (IgniteCache<int, EmpDataSet>): > Time for PutAll: 1924 ms for rows: 7500 > Time for fetching all data: 20288 ms for rows: 7500 > > There is a performance issue with DateTime value serialization. I'm > investigating and will get back to you with details soon. > > Meanwhile, there is a *workaround*: force Timestamp serialization format > and convert all values to UTC (which is a good idea anyway): > new IgniteClientConfiguration > { > ... > BinaryConfiguration = new BinaryConfiguration > { > Serializer = new BinaryReflectiveSerializer > { > ForceTimestamp = true > } > } > }; > > Then call ToUniversalTime() on all DateTime values. The result is: > Time for PutAll: 761 ms for rows: 7500 > Time for fetching all data: 645 ms for rows: 7500 > > For reference, JSON serialization of those 7500 rows (CreateMapFromList) > takes ~600ms on my machine. > > On Tue, Jan 14, 2020 at 4:09 PM Tunas <[email protected]> > wrote: > >> I am using Ignite 2.7.6 dll for Ignite and newtonSoft json for converting >> C# >> objects to json. >> Ignite configuration is same as mentioned allow. (Server and client are >> running on different machine) >> >> Sorry code class is little bit lengthy. >> >> One thing observed before publishing this code: addition of properties of >> type ObservableCollection<T> is taking more time. >> >> class Program >> { >> public static IIgniteClient IgniteThinClient; >> private static IgniteClientConfiguration >> GetIgniteClientConfiguration() >> { >> return new IgniteClientConfiguration >> { >> Endpoints = new[] { "MyWorkStation:10800" }, >> SocketTimeout = TimeSpan.FromSeconds(60) >> }; >> } >> static void Main(string[] args) >> { >> int rows = 7500; >> var lst = GetData(rows); >> >> Ignition.ClientMode = true; >> IgniteThinClient = >> Ignition.StartClient(GetIgniteClientConfiguration()); >> var cache = IgniteThinClient.GetOrCreateCache<int, >> string>("TestSlowness"); >> >> var map1 = CreateMapFromList(lst); >> Stopwatch sw = new Stopwatch(); >> sw.Start(); >> cache.PutAll(map1); >> sw.Stop(); >> Console.WriteLine($"Time for PutAll: {sw.ElapsedMilliseconds} >> ms >> for rows: {rows}"); >> sw.Reset(); >> >> >> ICacheClient<int, string> cacheClient = >> IgniteThinClient.GetOrCreateCache<int, string>("TestSlowness"); >> if(cacheClient!=null) >> { >> sw.Start(); >> *var cacheEntries = cacheClient.Query(new >> Apache.Ignite.Core.Cache.Query.ScanQuery<int, string>(null)).GetAll();* >> sw.Stop(); >> Console.WriteLine($"Time for fetching all data: >> {sw.ElapsedMilliseconds} ms for rows: {rows}"); >> } >> } >> >> //Actually below one is generic method(for simplicity changed to >> non-generic) >> private static ConcurrentDictionary<int, string> >> CreateMapFromList(IEnumerable<EmpDataSet> collection) >> { >> var map = new ConcurrentDictionary<int, string>(); >> if (collection != null && collection.Any()) >> { >> foreach (EmpDataSet item in collection) >> { >> var val = SerializedEntity(item); >> if (!string.IsNullOrWhiteSpace(val)) >> { >> map.TryAdd(item.DtId, val); >> } >> } >> } >> return map; >> } >> >> >> private static List<EmpDataSet> GetData(int rows = 7000) >> { >> Random rand = new Random(); >> >> var lst = new List<EmpDataSet>(); >> for (int i=1; i<=rows;i++) >> { >> var obj1 = new EmpDataSet(); >> obj1.DtId = i; >> obj1.EmpRef = rand.Next(rows + rows); >> obj1.DtName = RandomString(rand.Next(15, 20)); >> obj1.DraftDtId = rand.Next(rows + rows); >> obj1.ApprovedDtId = rand.Next(rows + rows); >> obj1.Description = RandomString(rand.Next(20, 60)); >> obj1.Code1 = RandomString(rand.Next(1, 10), >> Convert.ToBoolean(rand.Next(0, 1))); >> obj1.Code2 = RandomString(rand.Next(1, 10), >> Convert.ToBoolean(rand.Next(0, 1))); >> obj1.Label = RandomString(rand.Next(1, 10), >> Convert.ToBoolean(rand.Next(0, 1))); >> obj1.Etype = (EType)(rand.Next(1, 8)); >> obj1.Estatus = (EStatus)(rand.Next(1, 4)); >> obj1.Notes = RandomString(30, >> Convert.ToBoolean(rand.Next(0, >> 1))); >> obj1.Flag1 = Convert.ToBoolean(rand.Next(0, 1)); >> obj1.Flag2 = Convert.ToBoolean(rand.Next(0, 1)); >> obj1.Flag3 = Convert.ToBoolean(rand.Next(0, 1)); >> obj1.EmpAreaMap = ConstructEmpAreaMap(rand); >> obj1.EcType = (EcType)(rand.Next(1, 2)); >> obj1.Ref1 = rand.Next(50, 100); >> obj1.Ref1Name = RandomString(10, >> Convert.ToBoolean(rand.Next(0, 1))); >> obj1.DgId = rand.Next(24, 48); >> obj1.EmpRegs = GetEmpRegs(rand); >> obj1.DRegs = null; >> obj1.Fmap = new FMap { Id = rand.Next(1, rows + rows), >> Flts >> = ConstructFlts(rand, rand.Next(1, 4)) }; >> obj1.group1 = ConstructAgroup(rand); >> obj1.group2 = ConstructAgroup(rand); >> obj1.Str1 = RandomString(rand.Next(1, 50), >> Convert.ToBoolean(rand.Next(0, 1))); >> obj1.Str2 = RandomString(rand.Next(1, 50), >> Convert.ToBoolean(rand.Next(0, 1))); >> obj1.Str3 = RandomString(rand.Next(1, 50), >> Convert.ToBoolean(rand.Next(0, 1))); >> obj1.Str4 = RandomString(rand.Next(1, 50), >> Convert.ToBoolean(rand.Next(0, 1))); >> obj1.Str5 = RandomString(rand.Next(1, 50), >> Convert.ToBoolean(rand.Next(0, 1))); >> lst.Add(obj1); >> } >> return lst; >> } >> >> private static Agroup ConstructAgroup(Random rand) >> { >> int rows = rand.Next(10000, 20000); >> var obj1 = new Agroup >> { >> Id = rand.Next(1, rows + rows), >> Ref = RandomString(rand.Next(1, 40), >> Convert.ToBoolean(rand.Next(0, 1))), >> Flts = ConstructFlts(rand, rand.Next(1, 4)), >> EmpRegs = new >> ObservableCollection<EmpReg>(GetEmpRegs(rand)) >> }; >> return obj1; >> } >> >> private static Dictionary<EmpArea, List<EmpArea>> >> ConstructEmpAreaMap(Random rand, >> int rows=2, int innerCollection=3) >> { >> var map = new Dictionary<EmpArea, List<EmpArea>>(); >> for(int t=1; t<=rows;t++) >> { >> var obj2 = ConstructEmpAreaObj(rand); >> var lst = new List<EmpArea>(); >> for (int i = 1; i <= innerCollection; i++) >> { >> var obj1 = ConstructEmpAreaObj(rand); >> lst.Add(obj1); >> } >> map.Add(obj2, lst); >> } >> return map; >> } >> >> private static EmpArea ConstructEmpAreaObj(Random rand) >> { >> var obj1 = new EmpArea(); >> obj1.Id = rand.Next(1, 10); >> obj1.Name = RandomString(25); >> obj1.Description = RandomString(50); >> obj1.Freq = rand.Next(1, 100); >> obj1.StartNum = rand.Next(1, 50); >> return obj1; >> } >> >> private static ObservableCollection<Flt> ConstructFlts(Random >> rand, >> int rows=3) >> { >> var lst = new ObservableCollection<Flt>(); >> for(int i=0;i<rows;i++) >> { >> var obj = new Flt(); >> obj.Id1 = rand.Next(1, 50); >> obj.Id2 = rand.Next(1, 50); >> obj.Id3 = rand.Next(1, 50); >> obj.StartDate = DateTime.Today.AddDays(rand.Next(100, >> 200)); >> obj.EndDate = DateTime.Today.AddDays(rand.Next(1, 100)); >> obj.Share = Convert.ToDecimal(rand.NextDouble()); >> obj.Str1 = RandomString(rand.Next(10, 30), >> Convert.ToBoolean(rand.Next(0,1))); >> obj.Str2 = RandomString(rand.Next(10, 30), >> Convert.ToBoolean(rand.Next(0, 1))); >> obj.Str3 = RandomString(rand.Next(10, 30), >> Convert.ToBoolean(rand.Next(0, 1))); >> lst.Add(obj); >> } >> return lst; >> } >> >> private static List<EmpReg> GetEmpRegs(Random rand, int rows=2) >> { >> rows = rand.Next(1, 4); >> var lst = new List<EmpReg>(); >> for(int i=0;i<rows;i++) >> { >> var obj = new EmpReg(); >> obj.StartDate = DateTime.Today.AddDays(-1 * rand.Next(500, >> 1000)); >> obj.EndDate = DateTime.Today.AddDays(-1 * rand.Next(1, >> 500)); >> obj.InceptionDate = DateTime.Today.AddDays(-1 * >> rand.Next(1000, 1100)); >> obj.VDate = DateTime.Today.AddDays(-1 * rand.Next(1, 10)); >> obj.Id = rand.Next(1, 100); >> obj.Name = RandomString(20, Convert.ToBoolean(rand.Next(0, >> 1))); >> obj.Order = rand.Next(1000, 2000); >> lst.Add(obj); >> } >> return lst; >> } >> private static string RandomString(int size, bool lowerCase=false) >> { >> StringBuilder builder = new StringBuilder(); >> Random random = new Random(); >> char ch; >> for (int i = 0; i < size; i++) >> { >> ch = Convert.ToChar(Convert.ToInt32(Math.Floor(26 * >> random.NextDouble() + 65))); >> builder.Append(ch); >> } >> if (lowerCase) >> return builder.ToString().ToLower(); >> return builder.ToString(); >> } >> private static string SerializedEntity<T>(T value) >> { >> try >> { >> var settings = new JsonSerializerSettings >> { >> PreserveReferencesHandling = >> PreserveReferencesHandling.Objects, >> NullValueHandling = NullValueHandling.Ignore, >> ReferenceLoopHandling = >> ReferenceLoopHandling.Serialize, >> }; >> settings.ReferenceLoopHandling = >> ReferenceLoopHandling.Serialize; >> var json = JsonConvert.SerializeObject(value, settings); >> return json; >> } >> catch (Exception ex) >> { >> Console.WriteLine(ex); >> } >> return string.Empty; >> } >> } >> >> >> >> public class EmpDataSet >> { >> public int DtId { get; set; } >> public int EmpRef { get; set; } >> public string DtName { get; set; } >> public int? DraftDtId { get; set; } >> public int? ApprovedDtId { get; set; } >> public string Description { get; set; } >> public string Code1 { get; set; } >> public string Code2 { get; set; } >> public string Label { get; set; } >> public EType Etype { get; set; } >> public EStatus Estatus { get; set; } >> public string Notes { get; set; } >> public bool Flag1 { get; set; } >> public bool Flag2 { get; set; } >> public bool Flag3 { get; set; } >> public Dictionary<EmpArea, List<EmpArea>> EmpAreaMap { get; >> set; >> } >> public EcType? EcType { get; set; } >> public int? Ref1 { get; set; } >> public string Ref1Name { get; set; } >> public int? DgId { get; set; } >> private List<EmpReg> _empRegs; >> public List<EmpReg> EmpRegs >> { >> get => _empRegs ?? (_empRegs = new List<EmpReg>()); >> set => _empRegs = value; >> } >> private List<EmpReg> _dRegs; >> public List<EmpReg> DRegs >> { >> get => _dRegs ?? (_dRegs = new List<EmpReg>()); >> set => _dRegs = value; >> } >> public FMap Fmap { get; set; } >> public Agroup group1 { get; set; } >> public Agroup group2 { get; set; } >> public string Str1 { get; set; } >> public string Str2 { get; set; } >> public string Str3 { get; set; } >> public string Str4 { get; set; } >> public string Str5 { get; set; } >> >> } >> public class EmpArea >> { >> public int Id { get; set; } >> public string Name { get; set; } >> public string Description { get; set; } >> public int Freq { get; set; } >> public int StartNum { get; set; } >> } >> public enum EType >> { >> A=1, >> B=2, >> C=3, >> D=4, >> E=5, >> F=6, >> G=7, >> H=8 >> } >> public enum EStatus >> { >> A1=1, >> A2=2, >> A3=3, >> A4=4 >> } >> public enum EcType >> { >> S=1, >> M=2 >> } >> public class EmpReg >> { >> public DateTime? StartDate { get; set; } >> public DateTime? EndDate { get; set; } >> public DateTime? InceptionDate { get; set; } >> public DateTime? VDate { get; set; } >> public int Id { get; set; } >> public string Name { get; set; } >> public int Order { get; set; } >> >> } >> public class FMap >> { >> public int Id { get; set; } >> public ObservableCollection<Flt> Flts{ get; set; } >> } >> public class Agroup >> { >> public int Id { get; set; } >> public string Ref { get; set; } >> public ObservableCollection<Flt> Flts { get; set; } >> * public ObservableCollection<EmpReg> EmpRegs { get; set; }* >> } >> public class Flt >> { >> public int Id1 { get; set; } >> public int Id2 { get; set; } >> public int Id3 { get; set; } >> public DateTime? StartDate { get; set; } >> public DateTime? EndDate { get; set; } >> public decimal Share { get; set; } >> public string Str1 { get; set; } >> public string Str2 { get; set; } >> public string Str3 { get; set; } >> } >> >> >> >> -- >> Sent from: http://apache-ignite-users.70518.x6.nabble.com/ >> >
