Author: mordante
Date: Sat Mar 24 09:29:02 2012
New Revision: 53625
URL: http://svn.gna.org/viewcvs/wesnoth?rev=53625&view=rev
Log:
Let the tokenizer use the new buffered istream.
It already had a initial incarnation of the code, but now switches to
the class. In fact the class was based on this code.
Modified:
trunk/src/serialization/tokenizer.cpp
trunk/src/serialization/tokenizer.hpp
Modified: trunk/src/serialization/tokenizer.cpp
URL:
http://svn.gna.org/viewcvs/wesnoth/trunk/src/serialization/tokenizer.cpp?rev=53625&r1=53624&r2=53625&view=diff
==============================================================================
--- trunk/src/serialization/tokenizer.cpp (original)
+++ trunk/src/serialization/tokenizer.cpp Sat Mar 24 09:29:02 2012
@@ -29,11 +29,7 @@
textdomain_(PACKAGE),
file_(),
token_(),
- in_(in),
- cache_(),
- cache_size_(0),
- cache_offset_(0),
- read_eof_(false)
+ in_(in)
{
for (int c = 0; c < 128; ++c)
{
@@ -47,15 +43,14 @@
}
char_types_[c] = t;
}
- in_.exceptions(std::ios_base::badbit);
- fill_cache();
+ in_.stream().exceptions(std::ios_base::badbit);
next_char_fast();
}
tokenizer::~tokenizer()
{
- in_.clear(std::ios_base::goodbit);
- in_.exceptions(std::ios_base::goodbit);
+ in_.stream().clear(std::ios_base::goodbit);
+ in_.stream().exceptions(std::ios_base::goodbit);
}
const token &tokenizer::next_token()
Modified: trunk/src/serialization/tokenizer.hpp
URL:
http://svn.gna.org/viewcvs/wesnoth/trunk/src/serialization/tokenizer.hpp?rev=53625&r1=53624&r2=53625&view=diff
==============================================================================
--- trunk/src/serialization/tokenizer.hpp (original)
+++ trunk/src/serialization/tokenizer.hpp Sat Mar 24 09:29:02 2012
@@ -17,7 +17,7 @@
#ifndef TOKENIZER_H_INCLUDED
#define TOKENIZER_H_INCLUDED
-#include "util.hpp"
+#include "buffered_istream.hpp"
#include <istream>
#include <string>
@@ -102,47 +102,10 @@
next_char_fast();
}
- /**
- * Fills the cache.
- *
- * @warning This function must be called before @ref peek_char(). (Not
- * required before @ref next_char_fast(), but it can't hurt.) It must
also
- * be called after consuming a character from the cache.
- */
- void fill_cache()
- {
- if(UNLIKELY(cache_offset_ >= cache_size_)) {
- /*
- * This does not only test for the EOF, but also makes
sure the
- * data is available in the buffer. Without it readsome
will read
- * nothing, after its first call, even if the EOF has
not been
- * reached.
- */
- if(UNLIKELY(in_.rdbuf()->sgetc() == EOF)) {
- read_eof_ = true;
- } else {
- cache_offset_ = 0;
- cache_size_ = in_.readsome(cache_,
sizeof(cache_));
- }
- }
- }
-
void next_char_fast()
{
do {
- if(UNLIKELY(read_eof_)) {
- current_ = EOF;
- } else {
- /*
- * The data needs to be casted to an unsigned
value before it
- * is promoted to an int. The char might be
signed and contain
- * a negative value, resulting in a negative
result, and cause
- * problems. (Using gcc on x86 has this issue.)
- */
- current_ = static_cast<unsigned
char>(cache_[cache_offset_]);
- ++cache_offset_;
- fill_cache();
- }
+ current_ = in_.get();
} while (UNLIKELY(current_ == '\r'));
#if 0
/// @todo disabled untill campaign server is fixed
@@ -163,14 +126,9 @@
#endif
}
- int peek_char() const
+ int peek_char()
{
- if(UNLIKELY(read_eof_)) {
- return EOF;
- } else {
- /* See next_char_fast() */
- return static_cast<unsigned
char>(cache_[cache_offset_]);
- }
+ return in_.peek();
}
enum
@@ -214,36 +172,8 @@
#ifdef DEBUG
token previous_token_;
#endif
- std::istream &in_;
+ buffered_istream in_;
char char_types_[128];
-
- /**
- * Read cache.
- *
- * Reading from @ref std::istream isn't to fast, especially not a byte
at a
- * time. This cache is used to buffer x bytes at a time. The size of the
- * buffer is determined experimentally.
- */
- char cache_[1024];
-
- /**
- * The size of @ref cache_.
- *
- * When buffering the data there might be less data in the stream as in
the
- * buffer. This variable contains the exact size of the buffer.
- */
- unsigned cache_size_;
-
- /**
- * The offset of the current character in the buffer.
- *
- * @ref cache_[cache_offset_] is the current character, and can be
peaked
- * or consumed.
- */
- unsigned cache_offset_;
-
- /** Is the end of file reached? */
- bool read_eof_;
};
#endif
_______________________________________________
Wesnoth-commits mailing list
[email protected]
https://mail.gna.org/listinfo/wesnoth-commits
