Fix is complete and will be included in Ignite 2.8. Pre-release build is available right now: https://www.nuget.org/packages/Apache.Ignite/2.8.0-alpha20200122
On Mon, Jan 20, 2020 at 2:04 PM Pavel Tupitsyn <[email protected]> wrote: > Ticket filed and fix is on the way: > https://issues.apache.org/jira/browse/IGNITE-12555 > > On Tue, Jan 14, 2020 at 6:44 PM Pavel Tupitsyn <[email protected]> > wrote: > >> I've tried that code with a single local Ignite server, started with >> default configuration, the result is: >> Time for PutAll: 187 ms for rows: 7500 >> Time for fetching all data: 128 ms for rows: 7500 >> >> However, when I removed JSON serialization and stored the objects as is >> (IgniteCache<int, EmpDataSet>): >> Time for PutAll: 1924 ms for rows: 7500 >> Time for fetching all data: 20288 ms for rows: 7500 >> >> There is a performance issue with DateTime value serialization. I'm >> investigating and will get back to you with details soon. >> >> Meanwhile, there is a *workaround*: force Timestamp serialization format >> and convert all values to UTC (which is a good idea anyway): >> new IgniteClientConfiguration >> { >> ... >> BinaryConfiguration = new BinaryConfiguration >> { >> Serializer = new BinaryReflectiveSerializer >> { >> ForceTimestamp = true >> } >> } >> }; >> >> Then call ToUniversalTime() on all DateTime values. The result is: >> Time for PutAll: 761 ms for rows: 7500 >> Time for fetching all data: 645 ms for rows: 7500 >> >> For reference, JSON serialization of those 7500 rows (CreateMapFromList) >> takes ~600ms on my machine. >> >> On Tue, Jan 14, 2020 at 4:09 PM Tunas <[email protected]> >> wrote: >> >>> I am using Ignite 2.7.6 dll for Ignite and newtonSoft json for >>> converting C# >>> objects to json. >>> Ignite configuration is same as mentioned allow. (Server and client are >>> running on different machine) >>> >>> Sorry code class is little bit lengthy. >>> >>> One thing observed before publishing this code: addition of properties of >>> type ObservableCollection<T> is taking more time. >>> >>> class Program >>> { >>> public static IIgniteClient IgniteThinClient; >>> private static IgniteClientConfiguration >>> GetIgniteClientConfiguration() >>> { >>> return new IgniteClientConfiguration >>> { >>> Endpoints = new[] { "MyWorkStation:10800" }, >>> SocketTimeout = TimeSpan.FromSeconds(60) >>> }; >>> } >>> static void Main(string[] args) >>> { >>> int rows = 7500; >>> var lst = GetData(rows); >>> >>> Ignition.ClientMode = true; >>> IgniteThinClient = >>> Ignition.StartClient(GetIgniteClientConfiguration()); >>> var cache = IgniteThinClient.GetOrCreateCache<int, >>> string>("TestSlowness"); >>> >>> var map1 = CreateMapFromList(lst); >>> Stopwatch sw = new Stopwatch(); >>> sw.Start(); >>> cache.PutAll(map1); >>> sw.Stop(); >>> Console.WriteLine($"Time for PutAll: >>> {sw.ElapsedMilliseconds} ms >>> for rows: {rows}"); >>> sw.Reset(); >>> >>> >>> ICacheClient<int, string> cacheClient = >>> IgniteThinClient.GetOrCreateCache<int, string>("TestSlowness"); >>> if(cacheClient!=null) >>> { >>> sw.Start(); >>> *var cacheEntries = cacheClient.Query(new >>> Apache.Ignite.Core.Cache.Query.ScanQuery<int, string>(null)).GetAll();* >>> sw.Stop(); >>> Console.WriteLine($"Time for fetching all data: >>> {sw.ElapsedMilliseconds} ms for rows: {rows}"); >>> } >>> } >>> >>> //Actually below one is generic method(for simplicity changed to >>> non-generic) >>> private static ConcurrentDictionary<int, string> >>> CreateMapFromList(IEnumerable<EmpDataSet> collection) >>> { >>> var map = new ConcurrentDictionary<int, string>(); >>> if (collection != null && collection.Any()) >>> { >>> foreach (EmpDataSet item in collection) >>> { >>> var val = SerializedEntity(item); >>> if (!string.IsNullOrWhiteSpace(val)) >>> { >>> map.TryAdd(item.DtId, val); >>> } >>> } >>> } >>> return map; >>> } >>> >>> >>> private static List<EmpDataSet> GetData(int rows = 7000) >>> { >>> Random rand = new Random(); >>> >>> var lst = new List<EmpDataSet>(); >>> for (int i=1; i<=rows;i++) >>> { >>> var obj1 = new EmpDataSet(); >>> obj1.DtId = i; >>> obj1.EmpRef = rand.Next(rows + rows); >>> obj1.DtName = RandomString(rand.Next(15, 20)); >>> obj1.DraftDtId = rand.Next(rows + rows); >>> obj1.ApprovedDtId = rand.Next(rows + rows); >>> obj1.Description = RandomString(rand.Next(20, 60)); >>> obj1.Code1 = RandomString(rand.Next(1, 10), >>> Convert.ToBoolean(rand.Next(0, 1))); >>> obj1.Code2 = RandomString(rand.Next(1, 10), >>> Convert.ToBoolean(rand.Next(0, 1))); >>> obj1.Label = RandomString(rand.Next(1, 10), >>> Convert.ToBoolean(rand.Next(0, 1))); >>> obj1.Etype = (EType)(rand.Next(1, 8)); >>> obj1.Estatus = (EStatus)(rand.Next(1, 4)); >>> obj1.Notes = RandomString(30, >>> Convert.ToBoolean(rand.Next(0, >>> 1))); >>> obj1.Flag1 = Convert.ToBoolean(rand.Next(0, 1)); >>> obj1.Flag2 = Convert.ToBoolean(rand.Next(0, 1)); >>> obj1.Flag3 = Convert.ToBoolean(rand.Next(0, 1)); >>> obj1.EmpAreaMap = ConstructEmpAreaMap(rand); >>> obj1.EcType = (EcType)(rand.Next(1, 2)); >>> obj1.Ref1 = rand.Next(50, 100); >>> obj1.Ref1Name = RandomString(10, >>> Convert.ToBoolean(rand.Next(0, 1))); >>> obj1.DgId = rand.Next(24, 48); >>> obj1.EmpRegs = GetEmpRegs(rand); >>> obj1.DRegs = null; >>> obj1.Fmap = new FMap { Id = rand.Next(1, rows + rows), >>> Flts >>> = ConstructFlts(rand, rand.Next(1, 4)) }; >>> obj1.group1 = ConstructAgroup(rand); >>> obj1.group2 = ConstructAgroup(rand); >>> obj1.Str1 = RandomString(rand.Next(1, 50), >>> Convert.ToBoolean(rand.Next(0, 1))); >>> obj1.Str2 = RandomString(rand.Next(1, 50), >>> Convert.ToBoolean(rand.Next(0, 1))); >>> obj1.Str3 = RandomString(rand.Next(1, 50), >>> Convert.ToBoolean(rand.Next(0, 1))); >>> obj1.Str4 = RandomString(rand.Next(1, 50), >>> Convert.ToBoolean(rand.Next(0, 1))); >>> obj1.Str5 = RandomString(rand.Next(1, 50), >>> Convert.ToBoolean(rand.Next(0, 1))); >>> lst.Add(obj1); >>> } >>> return lst; >>> } >>> >>> private static Agroup ConstructAgroup(Random rand) >>> { >>> int rows = rand.Next(10000, 20000); >>> var obj1 = new Agroup >>> { >>> Id = rand.Next(1, rows + rows), >>> Ref = RandomString(rand.Next(1, 40), >>> Convert.ToBoolean(rand.Next(0, 1))), >>> Flts = ConstructFlts(rand, rand.Next(1, 4)), >>> EmpRegs = new >>> ObservableCollection<EmpReg>(GetEmpRegs(rand)) >>> }; >>> return obj1; >>> } >>> >>> private static Dictionary<EmpArea, List<EmpArea>> >>> ConstructEmpAreaMap(Random rand, >>> int rows=2, int innerCollection=3) >>> { >>> var map = new Dictionary<EmpArea, List<EmpArea>>(); >>> for(int t=1; t<=rows;t++) >>> { >>> var obj2 = ConstructEmpAreaObj(rand); >>> var lst = new List<EmpArea>(); >>> for (int i = 1; i <= innerCollection; i++) >>> { >>> var obj1 = ConstructEmpAreaObj(rand); >>> lst.Add(obj1); >>> } >>> map.Add(obj2, lst); >>> } >>> return map; >>> } >>> >>> private static EmpArea ConstructEmpAreaObj(Random rand) >>> { >>> var obj1 = new EmpArea(); >>> obj1.Id = rand.Next(1, 10); >>> obj1.Name = RandomString(25); >>> obj1.Description = RandomString(50); >>> obj1.Freq = rand.Next(1, 100); >>> obj1.StartNum = rand.Next(1, 50); >>> return obj1; >>> } >>> >>> private static ObservableCollection<Flt> ConstructFlts(Random >>> rand, >>> int rows=3) >>> { >>> var lst = new ObservableCollection<Flt>(); >>> for(int i=0;i<rows;i++) >>> { >>> var obj = new Flt(); >>> obj.Id1 = rand.Next(1, 50); >>> obj.Id2 = rand.Next(1, 50); >>> obj.Id3 = rand.Next(1, 50); >>> obj.StartDate = DateTime.Today.AddDays(rand.Next(100, >>> 200)); >>> obj.EndDate = DateTime.Today.AddDays(rand.Next(1, 100)); >>> obj.Share = Convert.ToDecimal(rand.NextDouble()); >>> obj.Str1 = RandomString(rand.Next(10, 30), >>> Convert.ToBoolean(rand.Next(0,1))); >>> obj.Str2 = RandomString(rand.Next(10, 30), >>> Convert.ToBoolean(rand.Next(0, 1))); >>> obj.Str3 = RandomString(rand.Next(10, 30), >>> Convert.ToBoolean(rand.Next(0, 1))); >>> lst.Add(obj); >>> } >>> return lst; >>> } >>> >>> private static List<EmpReg> GetEmpRegs(Random rand, int >>> rows=2) >>> { >>> rows = rand.Next(1, 4); >>> var lst = new List<EmpReg>(); >>> for(int i=0;i<rows;i++) >>> { >>> var obj = new EmpReg(); >>> obj.StartDate = DateTime.Today.AddDays(-1 * >>> rand.Next(500, >>> 1000)); >>> obj.EndDate = DateTime.Today.AddDays(-1 * rand.Next(1, >>> 500)); >>> obj.InceptionDate = DateTime.Today.AddDays(-1 * >>> rand.Next(1000, 1100)); >>> obj.VDate = DateTime.Today.AddDays(-1 * rand.Next(1, >>> 10)); >>> obj.Id = rand.Next(1, 100); >>> obj.Name = RandomString(20, >>> Convert.ToBoolean(rand.Next(0, >>> 1))); >>> obj.Order = rand.Next(1000, 2000); >>> lst.Add(obj); >>> } >>> return lst; >>> } >>> private static string RandomString(int size, bool >>> lowerCase=false) >>> { >>> StringBuilder builder = new StringBuilder(); >>> Random random = new Random(); >>> char ch; >>> for (int i = 0; i < size; i++) >>> { >>> ch = Convert.ToChar(Convert.ToInt32(Math.Floor(26 * >>> random.NextDouble() + 65))); >>> builder.Append(ch); >>> } >>> if (lowerCase) >>> return builder.ToString().ToLower(); >>> return builder.ToString(); >>> } >>> private static string SerializedEntity<T>(T value) >>> { >>> try >>> { >>> var settings = new JsonSerializerSettings >>> { >>> PreserveReferencesHandling = >>> PreserveReferencesHandling.Objects, >>> NullValueHandling = NullValueHandling.Ignore, >>> ReferenceLoopHandling = >>> ReferenceLoopHandling.Serialize, >>> }; >>> settings.ReferenceLoopHandling = >>> ReferenceLoopHandling.Serialize; >>> var json = JsonConvert.SerializeObject(value, settings); >>> return json; >>> } >>> catch (Exception ex) >>> { >>> Console.WriteLine(ex); >>> } >>> return string.Empty; >>> } >>> } >>> >>> >>> >>> public class EmpDataSet >>> { >>> public int DtId { get; set; } >>> public int EmpRef { get; set; } >>> public string DtName { get; set; } >>> public int? DraftDtId { get; set; } >>> public int? ApprovedDtId { get; set; } >>> public string Description { get; set; } >>> public string Code1 { get; set; } >>> public string Code2 { get; set; } >>> public string Label { get; set; } >>> public EType Etype { get; set; } >>> public EStatus Estatus { get; set; } >>> public string Notes { get; set; } >>> public bool Flag1 { get; set; } >>> public bool Flag2 { get; set; } >>> public bool Flag3 { get; set; } >>> public Dictionary<EmpArea, List<EmpArea>> EmpAreaMap { get; >>> set; >>> } >>> public EcType? EcType { get; set; } >>> public int? Ref1 { get; set; } >>> public string Ref1Name { get; set; } >>> public int? DgId { get; set; } >>> private List<EmpReg> _empRegs; >>> public List<EmpReg> EmpRegs >>> { >>> get => _empRegs ?? (_empRegs = new List<EmpReg>()); >>> set => _empRegs = value; >>> } >>> private List<EmpReg> _dRegs; >>> public List<EmpReg> DRegs >>> { >>> get => _dRegs ?? (_dRegs = new List<EmpReg>()); >>> set => _dRegs = value; >>> } >>> public FMap Fmap { get; set; } >>> public Agroup group1 { get; set; } >>> public Agroup group2 { get; set; } >>> public string Str1 { get; set; } >>> public string Str2 { get; set; } >>> public string Str3 { get; set; } >>> public string Str4 { get; set; } >>> public string Str5 { get; set; } >>> >>> } >>> public class EmpArea >>> { >>> public int Id { get; set; } >>> public string Name { get; set; } >>> public string Description { get; set; } >>> public int Freq { get; set; } >>> public int StartNum { get; set; } >>> } >>> public enum EType >>> { >>> A=1, >>> B=2, >>> C=3, >>> D=4, >>> E=5, >>> F=6, >>> G=7, >>> H=8 >>> } >>> public enum EStatus >>> { >>> A1=1, >>> A2=2, >>> A3=3, >>> A4=4 >>> } >>> public enum EcType >>> { >>> S=1, >>> M=2 >>> } >>> public class EmpReg >>> { >>> public DateTime? StartDate { get; set; } >>> public DateTime? EndDate { get; set; } >>> public DateTime? InceptionDate { get; set; } >>> public DateTime? VDate { get; set; } >>> public int Id { get; set; } >>> public string Name { get; set; } >>> public int Order { get; set; } >>> >>> } >>> public class FMap >>> { >>> public int Id { get; set; } >>> public ObservableCollection<Flt> Flts{ get; set; } >>> } >>> public class Agroup >>> { >>> public int Id { get; set; } >>> public string Ref { get; set; } >>> public ObservableCollection<Flt> Flts { get; set; } >>> * public ObservableCollection<EmpReg> EmpRegs { get; set; }* >>> } >>> public class Flt >>> { >>> public int Id1 { get; set; } >>> public int Id2 { get; set; } >>> public int Id3 { get; set; } >>> public DateTime? StartDate { get; set; } >>> public DateTime? EndDate { get; set; } >>> public decimal Share { get; set; } >>> public string Str1 { get; set; } >>> public string Str2 { get; set; } >>> public string Str3 { get; set; } >>> } >>> >>> >>> >>> -- >>> Sent from: http://apache-ignite-users.70518.x6.nabble.com/ >>> >>
