Re: [Kicad-developers] [PATCH] In-memory string io_benchmark and some docs
Hey John, Your updated patch builds on windows so I went ahead and merged it. Hopefully it wont be an issue on any other build platforms. Thanks, Wayne On 10/30/2018 7:31 AM, John Beard wrote: > Hi Wayne, > > On Mon, Oct 29, 2018 at 6:16 PM Wayne Stambaugh wrote: >> >> Patches 2 and 3 are merged. I merge the fixed patch when you post it. > > Freshly Jenkins'ed for your consideration. > > Cheers, > > John > ___ Mailing list: https://launchpad.net/~kicad-developers Post to : kicad-developers@lists.launchpad.net Unsubscribe : https://launchpad.net/~kicad-developers More help : https://help.launchpad.net/ListHelp
Re: [Kicad-developers] [PATCH] In-memory string io_benchmark and some docs
Hi Wayne,
On Mon, Oct 29, 2018 at 6:16 PM Wayne Stambaugh wrote:
>
> Patches 2 and 3 are merged. I merge the fixed patch when you post it.
Freshly Jenkins'ed for your consideration.
Cheers,
John
From be91dac4227ee6cce6e3188b3b8aebabfaa5efda Mon Sep 17 00:00:00 2001
From: John Beard
Date: Mon, 8 Oct 2018 19:11:46 +0100
Subject: [PATCH] Add an in-memory STREAM_LINE_READER benchmark
This adds an io-benchmark case of the STRING_LINE_READER
class, which reads a file into a std::string, *then*
reads it line by line.
As expected, due to it all being in memory, this is very fast.
Also fixes an issue in io_benchmark where the input file
must be in the current dir.
---
qa/qa_utils/stdstream_line_reader.cpp | 4 +-
tools/io_benchmark/io_benchmark.cpp | 75 ++-
2 files changed, 64 insertions(+), 15 deletions(-)
diff --git a/qa/qa_utils/stdstream_line_reader.cpp b/qa/qa_utils/stdstream_line_reader.cpp
index 3f6596fdb..d473f3442 100644
--- a/qa/qa_utils/stdstream_line_reader.cpp
+++ b/qa/qa_utils/stdstream_line_reader.cpp
@@ -68,7 +68,7 @@ void STDISTREAM_LINE_READER::SetStream( std::istream& aStream )
IFSTREAM_LINE_READER::IFSTREAM_LINE_READER( const wxFileName& aFileName ) :
-m_fStream( aFileName.GetFullName().ToUTF8() )
+m_fStream( aFileName.GetFullPath().ToUTF8() )
{
if( !m_fStream.is_open() )
{
@@ -79,7 +79,7 @@ IFSTREAM_LINE_READER::IFSTREAM_LINE_READER( const wxFileName& aFileName ) :
SetStream( m_fStream );
-m_source = aFileName.GetFullName();
+m_source = aFileName.GetFullPath();
}
diff --git a/tools/io_benchmark/io_benchmark.cpp b/tools/io_benchmark/io_benchmark.cpp
index 9442d9317..0c11a96c3 100644
--- a/tools/io_benchmark/io_benchmark.cpp
+++ b/tools/io_benchmark/io_benchmark.cpp
@@ -123,7 +123,7 @@ static void bench_line_reader( const wxFileName& aFile, int aReps, BENCH_REPORT&
{
for( int i = 0; i < aReps; ++i)
{
-LR fstr( aFile.GetFullName() );
+LR fstr( aFile.GetFullPath() );
while( fstr.ReadLine() )
{
report.linesRead++;
@@ -140,7 +140,7 @@ static void bench_line_reader( const wxFileName& aFile, int aReps, BENCH_REPORT&
template
static void bench_line_reader_reuse( const wxFileName& aFile, int aReps, BENCH_REPORT& report )
{
-LR fstr( aFile.GetFullName() );
+LR fstr( aFile.GetFullPath() );
for( int i = 0; i < aReps; ++i)
{
@@ -155,6 +155,53 @@ static void bench_line_reader_reuse( const wxFileName& aFile, int aReps, BENCH_R
}
+/**
+ * Benchmark using STRING_LINE_READER on string data read into memory from a file
+ * using std::ifstream, but read the data fresh from the file each time
+ */
+static void bench_string_lr( const wxFileName& aFile, int aReps, BENCH_REPORT& report )
+{
+for( int i = 0; i < aReps; ++i)
+{
+std::ifstream ifs( aFile.GetFullPath().ToStdString() );
+std::string content((std::istreambuf_iterator(ifs)),
+std::istreambuf_iterator());
+
+STRING_LINE_READER fstr( content, aFile.GetFullPath() );
+while( fstr.ReadLine() )
+{
+report.linesRead++;
+report.charAcc += (unsigned char) fstr.Line()[0];
+}
+}
+}
+
+
+/**
+ * Benchmark using STRING_LINE_READER on string data read into memory from a file
+ * using std::ifstream
+ *
+ * The STRING_LINE_READER is not reused (it cannot be rewound),
+ * but the file is read only once
+ */
+static void bench_string_lr_reuse( const wxFileName& aFile, int aReps, BENCH_REPORT& report )
+{
+std::ifstream ifs( aFile.GetFullPath().ToStdString() );
+std::string content((std::istreambuf_iterator(ifs)),
+std::istreambuf_iterator());
+
+for( int i = 0; i < aReps; ++i)
+{
+STRING_LINE_READER fstr( content, aFile.GetFullPath() );
+while( fstr.ReadLine() )
+{
+report.linesRead++;
+report.charAcc += (unsigned char) fstr.Line()[0];
+}
+}
+}
+
+
/**
* Benchmark using an INPUTSTREAM_LINE_READER with a given
* wxInputStream implementation.
@@ -163,11 +210,11 @@ static void bench_line_reader_reuse( const wxFileName& aFile, int aReps, BENCH_R
template
static void bench_wxis( const wxFileName& aFile, int aReps, BENCH_REPORT& report )
{
-S fileStream( aFile.GetFullName() );
+S fileStream( aFile.GetFullPath() );
for( int i = 0; i < aReps; ++i)
{
-INPUTSTREAM_LINE_READER istr( &fileStream, aFile.GetFullName() );
+INPUTSTREAM_LINE_READER istr( &fileStream, aFile.GetFullPath() );
while( istr.ReadLine() )
{
@@ -188,8 +235,8 @@ static void bench_wxis( const wxFileName& aFile, int aReps, BENCH_REPORT& report
template
static void bench_wxis_reuse( const wxFileName& aFile, int aReps, BENCH_REPORT& report )
{
-S fileStream( aFile.GetFullName() );
-INPUTSTREAM_LINE_READER istr( &fileStream, aFile.GetFullName() );
+S fileStream( aFile.GetFullPath() );
+I
Re: [Kicad-developers] [PATCH] In-memory string io_benchmark and some docs
John, Patches 2 and 3 are merged. I merge the fixed patch when you post it. Thanks, Wayne On 10/29/2018 2:10 PM, John Beard wrote: > Hi Wayne, > > Sure, that would be great. > > I'll get a new patch on Jenkins and get back to you with it for #1. I > thought it was already run though Jenkins before, but clearly not. > > Cheers, > > John > > On 29 October 2018 17:09:34 GMT+00:00, Wayne Stambaugh > wrote: > > John, > > Do you want me to go ahead and merge patches 2 and 3 since they are just > documentation changes? > > Cheers, > > Wayne > > On 10/29/2018 10:38 AM, John Beard wrote: > > Hi, > > Just a couple of tweaks to the io_benchmark, including a baseline > in-memory string-based streams, and some other dev-doc stuff > that was > conflicting with the fuzzing docs. > > Cheers, > > John > > > Mailing list: https://launchpad.net/~kicad-developers > Post to : kicad-developers@lists.launchpad.net > Unsubscribe : https://launchpad.net/~kicad-developers > More help : https://help.launchpad.net/ListHelp > > > Mailing list: https://launchpad.net/~kicad-developers > Post to : kicad-developers@lists.launchpad.net > Unsubscribe : https://launchpad.net/~kicad-developers > More help : https://help.launchpad.net/ListHelp > ___ Mailing list: https://launchpad.net/~kicad-developers Post to : kicad-developers@lists.launchpad.net Unsubscribe : https://launchpad.net/~kicad-developers More help : https://help.launchpad.net/ListHelp
Re: [Kicad-developers] [PATCH] In-memory string io_benchmark and some docs
Hi Wayne, Sure, that would be great. I'll get a new patch on Jenkins and get back to you with it for #1. I thought it was already run though Jenkins before, but clearly not. Cheers, John On 29 October 2018 17:09:34 GMT+00:00, Wayne Stambaugh wrote: >John, > >Do you want me to go ahead and merge patches 2 and 3 since they are >just >documentation changes? > >Cheers, > >Wayne > >On 10/29/2018 10:38 AM, John Beard wrote: >> Hi, >> >> Just a couple of tweaks to the io_benchmark, including a baseline >> in-memory string-based streams, and some other dev-doc stuff that was >> conflicting with the fuzzing docs. >> >> Cheers, >> >> John >> >> >> ___ >> Mailing list: https://launchpad.net/~kicad-developers >> Post to : kicad-developers@lists.launchpad.net >> Unsubscribe : https://launchpad.net/~kicad-developers >> More help : https://help.launchpad.net/ListHelp >> > >___ >Mailing list: https://launchpad.net/~kicad-developers >Post to : kicad-developers@lists.launchpad.net >Unsubscribe : https://launchpad.net/~kicad-developers >More help : https://help.launchpad.net/ListHelp ___ Mailing list: https://launchpad.net/~kicad-developers Post to : kicad-developers@lists.launchpad.net Unsubscribe : https://launchpad.net/~kicad-developers More help : https://help.launchpad.net/ListHelp
Re: [Kicad-developers] [PATCH] In-memory string io_benchmark and some docs
John, Do you want me to go ahead and merge patches 2 and 3 since they are just documentation changes? Cheers, Wayne On 10/29/2018 10:38 AM, John Beard wrote: > Hi, > > Just a couple of tweaks to the io_benchmark, including a baseline > in-memory string-based streams, and some other dev-doc stuff that was > conflicting with the fuzzing docs. > > Cheers, > > John > > > ___ > Mailing list: https://launchpad.net/~kicad-developers > Post to : kicad-developers@lists.launchpad.net > Unsubscribe : https://launchpad.net/~kicad-developers > More help : https://help.launchpad.net/ListHelp > ___ Mailing list: https://launchpad.net/~kicad-developers Post to : kicad-developers@lists.launchpad.net Unsubscribe : https://launchpad.net/~kicad-developers More help : https://help.launchpad.net/ListHelp
Re: [Kicad-developers] [PATCH] In-memory string io_benchmark and some docs
Sorry for the fat finger send. Her is the error: C:/msys64/home/wstambaugh/src/kicad-trunk/tools/io_benchmark/io_benchmark.cpp: In function 'void bench_string_lr(const wxFileName&, int, BENCH_REPORT&)': C:/msys64/home/wstambaugh/src/kicad-trunk/tools/io_benchmark/io_benchmark.cpp:166:48: error: call of overloaded 'basic_ifstream(wxString)' is ambiguous std::ifstream ifs( aFile.GetFullPath() ); ^ In file included from C:/msys64/home/wstambaugh/src/kicad-trunk/tools/io_benchmark/io_benchmark.cpp:32: C:/msys64/mingw64/include/c++/8.2.0/fstream:539:7: note: candidate: 'std::basic_ifstream<_CharT, _Traits>::basic_ifstream(const wchar_t*, std::ios_base::openmode) [with _CharT = char; _Traits = std::char_traits; std::ios_base::openmode = std::_Ios_Openmode]' basic_ifstream(const wchar_t* __s, ^~ C:/msys64/mingw64/include/c++/8.2.0/fstream:524:7: note: candidate: 'std::basic_ifstream<_CharT, _Traits>::basic_ifstream(const char*, std::ios_base::openmode)[with _CharT = char; _Traits = std::char_traits; std::ios_base::openmode = std::_Ios_Openmode]' basic_ifstream(const char* __s, ios_base::openmode __mode = ios_base::in) ^~ C:/msys64/home/wstambaugh/src/kicad-trunk/tools/io_benchmark/io_benchmark.cpp: In function 'void bench_string_lr_reuse(const wxFileName&, int, BENCH_REPORT&)': C:/msys64/home/wstambaugh/src/kicad-trunk/tools/io_benchmark/io_benchmark.cpp:189:44: error: call of overloaded 'basic_ifstream(wxString)' is ambiguous std::ifstream ifs( aFile.GetFullPath() ); ^ In file included from C:/msys64/home/wstambaugh/src/kicad-trunk/tools/io_benchmark/io_benchmark.cpp:32: C:/msys64/mingw64/include/c++/8.2.0/fstream:539:7: note: candidate: 'std::basic_ifstream<_CharT, _Traits>::basic_ifstream(const wchar_t*, std::ios_base::openmode) [with _CharT = char; _Traits = std::char_traits; std::ios_base::openmode = std::_Ios_Openmode]' basic_ifstream(const wchar_t* __s, ^~ C:/msys64/mingw64/include/c++/8.2.0/fstream:524:7: note: candidate: 'std::basic_ifstream<_CharT, _Traits>::basic_ifstream(const char*, std::ios_base::openmode)[with _CharT = char; _Traits = std::char_traits; std::ios_base::openmode = std::_Ios_Openmode]' basic_ifstream(const char* __s, ios_base::openmode __mode = ios_base::in) ^~ On 10/29/2018 10:38 AM, John Beard wrote: > Hi, > > Just a couple of tweaks to the io_benchmark, including a baseline > in-memory string-based streams, and some other dev-doc stuff that was > conflicting with the fuzzing docs. > > Cheers, > > John > > > ___ > Mailing list: https://launchpad.net/~kicad-developers > Post to : kicad-developers@lists.launchpad.net > Unsubscribe : https://launchpad.net/~kicad-developers > More help : https://help.launchpad.net/ListHelp > ___ Mailing list: https://launchpad.net/~kicad-developers Post to : kicad-developers@lists.launchpad.net Unsubscribe : https://launchpad.net/~kicad-developers More help : https://help.launchpad.net/ListHelp
Re: [Kicad-developers] [PATCH] In-memory string io_benchmark and some docs
John, Patch 1 fails to build on msys2/ming64 with the following error: On 10/29/2018 10:38 AM, John Beard wrote: > Hi, > > Just a couple of tweaks to the io_benchmark, including a baseline > in-memory string-based streams, and some other dev-doc stuff that was > conflicting with the fuzzing docs. > > Cheers, > > John > > > ___ > Mailing list: https://launchpad.net/~kicad-developers > Post to : kicad-developers@lists.launchpad.net > Unsubscribe : https://launchpad.net/~kicad-developers > More help : https://help.launchpad.net/ListHelp > ___ Mailing list: https://launchpad.net/~kicad-developers Post to : kicad-developers@lists.launchpad.net Unsubscribe : https://launchpad.net/~kicad-developers More help : https://help.launchpad.net/ListHelp
[Kicad-developers] [PATCH] In-memory string io_benchmark and some docs
Hi,
Just a couple of tweaks to the io_benchmark, including a baseline
in-memory string-based streams, and some other dev-doc stuff that was
conflicting with the fuzzing docs.
Cheers,
John
From 68955bbcb90c5fc12e4f3a61b6d7df8a2541c64b Mon Sep 17 00:00:00 2001
From: John Beard
Date: Mon, 8 Oct 2018 19:11:46 +0100
Subject: [PATCH 1/3] Add an in-memory STREAM_LINE_READER benchmark
This adds an io-benchmark case of the STRING_LINE_READER
class, which reads a file into a std::string, *then*
reads it line by line.
As expected, due to it all being in memory, this is very fast.
Also fixes an issue in io_benchmark where the input file
must be in the current dir.
---
qa/qa_utils/stdstream_line_reader.cpp | 4 +-
tools/io_benchmark/io_benchmark.cpp | 75 ++-
2 files changed, 64 insertions(+), 15 deletions(-)
diff --git a/qa/qa_utils/stdstream_line_reader.cpp b/qa/qa_utils/stdstream_line_reader.cpp
index 3f6596fdb..d473f3442 100644
--- a/qa/qa_utils/stdstream_line_reader.cpp
+++ b/qa/qa_utils/stdstream_line_reader.cpp
@@ -68,7 +68,7 @@ void STDISTREAM_LINE_READER::SetStream( std::istream& aStream )
IFSTREAM_LINE_READER::IFSTREAM_LINE_READER( const wxFileName& aFileName ) :
-m_fStream( aFileName.GetFullName().ToUTF8() )
+m_fStream( aFileName.GetFullPath().ToUTF8() )
{
if( !m_fStream.is_open() )
{
@@ -79,7 +79,7 @@ IFSTREAM_LINE_READER::IFSTREAM_LINE_READER( const wxFileName& aFileName ) :
SetStream( m_fStream );
-m_source = aFileName.GetFullName();
+m_source = aFileName.GetFullPath();
}
diff --git a/tools/io_benchmark/io_benchmark.cpp b/tools/io_benchmark/io_benchmark.cpp
index 9442d9317..7236884d6 100644
--- a/tools/io_benchmark/io_benchmark.cpp
+++ b/tools/io_benchmark/io_benchmark.cpp
@@ -123,7 +123,7 @@ static void bench_line_reader( const wxFileName& aFile, int aReps, BENCH_REPORT&
{
for( int i = 0; i < aReps; ++i)
{
-LR fstr( aFile.GetFullName() );
+LR fstr( aFile.GetFullPath() );
while( fstr.ReadLine() )
{
report.linesRead++;
@@ -140,7 +140,7 @@ static void bench_line_reader( const wxFileName& aFile, int aReps, BENCH_REPORT&
template
static void bench_line_reader_reuse( const wxFileName& aFile, int aReps, BENCH_REPORT& report )
{
-LR fstr( aFile.GetFullName() );
+LR fstr( aFile.GetFullPath() );
for( int i = 0; i < aReps; ++i)
{
@@ -155,6 +155,53 @@ static void bench_line_reader_reuse( const wxFileName& aFile, int aReps, BENCH_R
}
+/**
+ * Benchmark using STRING_LINE_READER on string data read into memory from a file
+ * using std::ifstream, but read the data fresh from the file each time
+ */
+static void bench_string_lr( const wxFileName& aFile, int aReps, BENCH_REPORT& report )
+{
+for( int i = 0; i < aReps; ++i)
+{
+std::ifstream ifs( aFile.GetFullPath() );
+std::string content((std::istreambuf_iterator(ifs)),
+std::istreambuf_iterator());
+
+STRING_LINE_READER fstr( content, aFile.GetFullPath() );
+while( fstr.ReadLine() )
+{
+report.linesRead++;
+report.charAcc += (unsigned char) fstr.Line()[0];
+}
+}
+}
+
+
+/**
+ * Benchmark using STRING_LINE_READER on string data read into memory from a file
+ * using std::ifstream
+ *
+ * The STRING_LINE_READER is not reused (it cannot be rewound),
+ * but the file is read only once
+ */
+static void bench_string_lr_reuse( const wxFileName& aFile, int aReps, BENCH_REPORT& report )
+{
+std::ifstream ifs( aFile.GetFullPath() );
+std::string content((std::istreambuf_iterator(ifs)),
+std::istreambuf_iterator());
+
+for( int i = 0; i < aReps; ++i)
+{
+STRING_LINE_READER fstr( content, aFile.GetFullPath() );
+while( fstr.ReadLine() )
+{
+report.linesRead++;
+report.charAcc += (unsigned char) fstr.Line()[0];
+}
+}
+}
+
+
/**
* Benchmark using an INPUTSTREAM_LINE_READER with a given
* wxInputStream implementation.
@@ -163,11 +210,11 @@ static void bench_line_reader_reuse( const wxFileName& aFile, int aReps, BENCH_R
template
static void bench_wxis( const wxFileName& aFile, int aReps, BENCH_REPORT& report )
{
-S fileStream( aFile.GetFullName() );
+S fileStream( aFile.GetFullPath() );
for( int i = 0; i < aReps; ++i)
{
-INPUTSTREAM_LINE_READER istr( &fileStream, aFile.GetFullName() );
+INPUTSTREAM_LINE_READER istr( &fileStream, aFile.GetFullPath() );
while( istr.ReadLine() )
{
@@ -188,8 +235,8 @@ static void bench_wxis( const wxFileName& aFile, int aReps, BENCH_REPORT& report
template
static void bench_wxis_reuse( const wxFileName& aFile, int aReps, BENCH_REPORT& report )
{
-S fileStream( aFile.GetFullName() );
-INPUTSTREAM_LINE_READER istr( &fileStream, aFile.GetFullName() );
+S fileStream( aFile.GetFullPath() );
+INPUTSTREAM_LINE_READER istr( &file
