jheight 2004/10/12 20:37:56
Modified: src/java/org/apache/poi/hssf/model Sheet.java
src/java/org/apache/poi/hssf/record
SSTRecordSizeCalculator.java SSTSerializer.java
src/testcases/org/apache/poi/hssf/record/aggregates
TestValueRecordsAggregate.java
Log:
PR: 19974
This patch fixes writeing VERY long > 8228 character strings. Existing SST
serialisation had problem with calculation and writing to multiple continue records.
Also fixed some test case errors introduced by my last patch for DBCell and
IndexRecords
Revision Changes Path
1.49 +3 -1 jakarta-poi/src/java/org/apache/poi/hssf/model/Sheet.java
Index: Sheet.java
===================================================================
RCS file: /home/cvs/jakarta-poi/src/java/org/apache/poi/hssf/model/Sheet.java,v
retrieving revision 1.48
retrieving revision 1.49
diff -u -r1.48 -r1.49
--- Sheet.java 7 Oct 2004 03:37:15 -0000 1.48
+++ Sheet.java 13 Oct 2004 03:37:56 -0000 1.49
@@ -763,7 +763,7 @@
if (record.getSid() == BOFRecord.sid) {
//Can there be more than one BOF for a sheet? If not then we can
//remove this guard. So be safe it is left here.
- if (!haveSerializedIndex) {
+ if ((rows != null) && (!haveSerializedIndex)) {
haveSerializedIndex = true;
pos += serializeIndexRecord(k, pos, data);
}
@@ -2029,6 +2029,7 @@
retval += (( Record ) records.get(k)).getRecordSize();
}
//Add space for the IndexRecord
+ if (rows != null) {
final int blocks = rows.getRowBlockCount();
retval += IndexRecord.getRecordSizeForBlockCount(blocks);
@@ -2042,6 +2043,7 @@
RowRecord row = (RowRecord)itr.next();
if (cells.rowHasCells(row.getRowNumber()))
retval += 2;
+ }
}
return retval;
}
1.7 +62 -165
jakarta-poi/src/java/org/apache/poi/hssf/record/SSTRecordSizeCalculator.java
Index: SSTRecordSizeCalculator.java
===================================================================
RCS file:
/home/cvs/jakarta-poi/src/java/org/apache/poi/hssf/record/SSTRecordSizeCalculator.java,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -r1.6 -r1.7
--- SSTRecordSizeCalculator.java 23 Aug 2004 08:52:34 -0000 1.6
+++ SSTRecordSizeCalculator.java 13 Oct 2004 03:37:56 -0000 1.7
@@ -30,6 +30,7 @@
* the SST serialization code needs to be rewritten.
*
* @author Glen Stampoultzis (glens at apache.org)
+ * @author Jason Height (jheight at apache.org)
*/
class SSTRecordSizeCalculator
{
@@ -52,180 +53,76 @@
this.strings = strings;
}
- /**
- * Calculate the size in bytes of the SST record. This will include continue
- * records.
- *
- * @return the size of the SST record.
- */
- public int getRecordSize()
- {
- initVars();
-
- int retval;
- int totalStringSpaceRequired = SSTSerializer.calculateUnicodeSize(strings);
-
- if ( totalStringSpaceRequired > SSTRecord.MAX_DATA_SPACE )
- {
- retval = sizeOverContinuation( totalStringSpaceRequired );
- }
- else
- {
- // short data: write one simple SST record
- retval = SSTRecord.SST_RECORD_OVERHEAD + totalStringSpaceRequired;
- recordLengths.add( new Integer( totalStringSpaceRequired ) );
- }
- return retval;
- }
-
- public List getRecordLengths()
- {
- return recordLengths;
- }
-
- private int sizeOverContinuation( int totalStringSpaceRequired )
- {
- int retval;
-
- while ( !finished )
- {
- recordSize = 0;
- pos = 0;
-
- if ( firstRecord )
- {
- addMaxLengthRecordSize();
- }
- else
- {
-
- // writing continue record
- pos = 0;
- int toBeWritten = ( totalStringSpaceRequired - totalBytesWritten )
+ ( isRemainingString ? 1 : 0 );
- int size = Math.min( SSTRecord.MAX_RECORD_SIZE -
SSTRecord.STD_RECORD_OVERHEAD, toBeWritten );
-
- if ( size == toBeWritten )
- {
- finished = true;
- }
- recordSize = size + SSTRecord.STD_RECORD_OVERHEAD;
- recordLengths.add( new Integer( size ) );
- pos = 4;
- }
- if ( isRemainingString )
- {
- calcReminant();
- }
- calcRemainingStrings();
- totalWritten += recordSize;
- }
- retval = totalWritten;
-
- return retval;
- }
-
- private void addMaxLengthRecordSize()
- {
- // writing SST record
- recordSize = SSTRecord.MAX_RECORD_SIZE;
- pos = 12;
- firstRecord = false;
- recordLengths.add( new Integer( recordSize - SSTRecord.STD_RECORD_OVERHEAD
) );
- }
-
- private void calcRemainingStrings()
- {
- for ( ; unipos < strings.size(); unipos++ )
- {
- int available = SSTRecord.MAX_RECORD_SIZE - pos;
- Integer intunipos = new Integer( unipos );
-
- unistr = ( (UnicodeString) strings.get( intunipos ) );
- if ( unistr.getRecordSize() <= available )
- {
- totalBytesWritten += unistr.getRecordSize();
- pos += unistr.getRecordSize();
- }
- else
- {
- if ( available >= SSTRecord.STRING_MINIMAL_OVERHEAD )
- {
- int toBeWritten =
- unistr.maxBrokenLength( available );
-
- totalBytesWritten += toBeWritten;
- stringReminant =
- ( unistr.getRecordSize() - toBeWritten )
- + LittleEndianConsts.BYTE_SIZE;
- if ( available != toBeWritten )
- {
- int shortrecord = recordSize
- - ( available - toBeWritten );
-
- recordLengths.set(
- recordLengths.size() - 1,
- new Integer(
- shortrecord - SSTRecord.STD_RECORD_OVERHEAD
) );
- recordSize = shortrecord;
+ private boolean canFitStringInRecord(int recordLength) {
+ return (recordLength+SSTRecord.STRING_MINIMAL_OVERHEAD) <
SSTRecord.MAX_RECORD_SIZE;
}
- isRemainingString = true;
- unipos++;
- }
- else
- {
- int shortrecord = recordSize - available;
- recordLengths.set( recordLengths.size() - 1,
- new Integer( shortrecord -
SSTRecord.STD_RECORD_OVERHEAD ) );
- recordSize = shortrecord;
- }
- break;
- }
+ public int getRecordSize() {
+ //Indicates how much of the current base or continue record has
+ //been written
+ int continueSize = SSTRecord.SST_RECORD_OVERHEAD;
+ int recordSize = 0;
+ for (int i=0; i < strings.size(); i++ )
+ {
+ Integer intunipos = new Integer(i);
+ UnicodeString unistr = ( (UnicodeString) strings.get(intunipos));
+ final int stringLength = unistr.getRecordSize();
+ if ((continueSize + stringLength) <= SSTRecord.MAX_RECORD_SIZE) {
+ //String can fit within the bounds of the current record (SST or
Continue)
+ continueSize += stringLength;
+
+ if ((i < (strings.size()-1)) && !canFitStringInRecord(continueSize)) {
+ //Start new continueRecord if there is another string
+ recordLengths.add(new Integer(continueSize));
+ recordSize += continueSize;
+ //Minimum ammount of space for a new continue record.
+ continueSize = 4;
+ }
+ } else {
+ int stringRemainder = stringLength;
+ while (stringRemainder != 0) {
+ if ( (continueSize + stringRemainder) > SSTRecord.MAX_RECORD_SIZE) {
+ //Determine number of bytes that can be written in the space
+ //available
+ int bytesWritten = Math.min((SSTRecord.MAX_RECORD_SIZE -
continueSize), stringRemainder);
+
+ //Ensure that the Unicode String writes both the high and low
+ //byte in the one action. Since the string overhead is 3 bytes
+ //if the bytes that can be written is even, then we need to
+ //write one less byte to capture both the high and low bytes.
+ bytesWritten = unistr.maxBrokenLength(bytesWritten);
+ continueSize += bytesWritten;
+ stringRemainder -= bytesWritten;
+ recordLengths.add(new Integer(continueSize));
+ recordSize += continueSize;
+ //Minimum ammount of space for a new continue record.
+ continueSize = 4;
+ //Add one to the size of the string that is remaining, since the
+ //first byte for the next continue record will be compressed
unicode indicator
+ stringRemainder++;
+ } else {
+ //Remainder of string can fit within the bounds of the current
+ //continue record
+ continueSize += stringRemainder;
+ stringRemainder = 0;
+ if ((i < (strings.size()-1)) &&
!canFitStringInRecord(continueSize)) {
+ //Start new continueRecord if there is another string
+ recordLengths.add(new Integer(continueSize));
+ recordSize += continueSize;
+ //Minimum ammount of space for a new continue record.
+ continueSize = 4;
}
}
-
- private void calcReminant()
- {
- int available = SSTRecord.MAX_RECORD_SIZE - pos;
-
- if ( stringReminant <= available )
- {
-
- // write reminant
- totalBytesWritten += stringReminant - 1;
- pos += stringReminant;
- isRemainingString = false;
}
- else
- {
-
- // write as much of the remnant as possible
- int toBeWritten = unistr.maxBrokenLength( available );
-
- if ( available != toBeWritten )
- {
- int shortrecord = recordSize - ( available - toBeWritten );
- recordLengths.set( recordLengths.size() - 1,
- new Integer( shortrecord - SSTRecord.STD_RECORD_OVERHEAD )
);
- recordSize = shortrecord;
}
- totalBytesWritten += toBeWritten - 1;
- pos += toBeWritten;
- stringReminant -= toBeWritten - 1;
- isRemainingString = true;
}
+ recordLengths.add(new Integer(continueSize));
+ recordSize += continueSize;
+ return recordSize;
}
- private void initVars()
+ public List getRecordLengths()
{
- unistr = null;
- stringReminant = 0;
- unipos = 0;
- isRemainingString = false;
- totalBytesWritten = 0;
- finished = false;
- firstRecord = true;
- totalWritten = 0;
+ return recordLengths;
}
-
}
1.12 +13 -5
jakarta-poi/src/java/org/apache/poi/hssf/record/SSTSerializer.java
Index: SSTSerializer.java
===================================================================
RCS file:
/home/cvs/jakarta-poi/src/java/org/apache/poi/hssf/record/SSTSerializer.java,v
retrieving revision 1.11
retrieving revision 1.12
diff -u -r1.11 -r1.12
--- SSTSerializer.java 23 Aug 2004 08:52:34 -0000 1.11
+++ SSTSerializer.java 13 Oct 2004 03:37:56 -0000 1.12
@@ -111,7 +111,7 @@
private void serializeSingleSSTRecord( byte[] data, int offset, int
record_length_index )
{
int len = ( (Integer) recordLengths.get( record_length_index ) ).intValue();
- int recordSize = SSTRecord.SST_RECORD_OVERHEAD + len -
SSTRecord.STD_RECORD_OVERHEAD;
+ int recordSize = len - SSTRecord.STD_RECORD_OVERHEAD;
sstRecordHeader.writeSSTHeader( data, 0 + offset, recordSize );
int pos = SSTRecord.SST_RECORD_OVERHEAD;
@@ -149,13 +149,16 @@
while ( totalWritten != record_size )
{
- int recordLength = ( (Integer) recordLengths.get( record_length_index++
) ).intValue();
+ //Total record length, including excel record header and sst/continue
header
+ final int recordLength = ( (Integer) recordLengths.get(
record_length_index++ ) ).intValue();
+ //Total available data length (minus the excel record header size)
+ final int recordDataLength = recordLength - 4;
RecordProcessor recordProcessor = new RecordProcessor( buffer,
- recordLength, numStrings, numUniqueStrings );
+ recordDataLength, numStrings, numUniqueStrings );
// write the appropriate header
startOfRecord = offset + totalWritten;
- recordProcessor.writeRecordHeader( offset, totalWritten, recordLength,
first_record );
+ recordProcessor.writeRecordHeader( offset, totalWritten,
recordDataLength, first_record );
first_record = false;
// now, write the rest of the data into the current
@@ -166,6 +169,11 @@
// the last string in the previous record was not written out
completely
stringReminant = recordProcessor.writeStringRemainder(
lastneedcontinue,
stringReminant, offset, totalWritten );
+ //Check to see if still not written out completely
+ if (lastneedcontinue) {
+ totalWritten += recordLength;
+ continue;
+ }
}
// last string's remnant, if any, is cleaned up as best as can be done
... now let's try and write
@@ -204,7 +212,7 @@
break;
}
}
- totalWritten += recordLength + SSTRecord.STD_RECORD_OVERHEAD;
+ totalWritten += recordLength;
}
}
1.6 +2 -1
jakarta-poi/src/testcases/org/apache/poi/hssf/record/aggregates/TestValueRecordsAggregate.java
Index: TestValueRecordsAggregate.java
===================================================================
RCS file:
/home/cvs/jakarta-poi/src/testcases/org/apache/poi/hssf/record/aggregates/TestValueRecordsAggregate.java,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -r1.5 -r1.6
--- TestValueRecordsAggregate.java 23 Aug 2004 08:52:48 -0000 1.5
+++ TestValueRecordsAggregate.java 13 Oct 2004 03:37:56 -0000 1.6
@@ -161,7 +161,8 @@
};
List records = testData();
valueRecord.construct( 0, records );
- int bytesWritten = valueRecord.serialize( 0, actualArray );
+ int bytesWritten = valueRecord.serializeCellRow(1, 0, actualArray );
+ bytesWritten += valueRecord.serializeCellRow(2, bytesWritten, actualArray );
assertEquals( 36, bytesWritten );
for (int i = 0; i < 36; i++)
assertEquals( expectedArray[i], actualArray[i] );
---------------------------------------------------------------------
To unsubscribe, e-mail: [EMAIL PROTECTED]
For additional commands, e-mail: [EMAIL PROTECTED]
