This is an automated email from the ASF dual-hosted git repository. leirui pushed a commit to branch research/M4-visualization in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 082283ef4d16721c27d7a8eb48f3225eb3aabcb2 Author: Lei Rui <[email protected]> AuthorDate: Mon Oct 10 15:39:59 2022 +0800 width 8 multiple faster, otherwise no --- .../iotdb/session/MyBasicOperationTest2.java | 16 +- ...rationTest2.java => MyBasicOperationTest3.java} | 57 +++--- .../session/MyRealDataTest1_WriteAndQuery.java | 94 ++++----- .../tsfile/common/constant/TsFileConstant.java | 26 ++- .../encoding/decoder/DeltaBinaryDecoder.java | 42 ++-- .../org/apache/iotdb/tsfile/utils/BytesUtils.java | 211 +++++++++++++-------- 6 files changed, 272 insertions(+), 174 deletions(-) diff --git a/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest2.java b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest2.java index eed2b00e6e..819054b5e4 100644 --- a/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest2.java +++ b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest2.java @@ -11,7 +11,7 @@ public class MyBasicOperationTest2 { // op1: long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i, packWidth); // op2: put bytes as a whole into long - int repeat = 1000000; + int repeat = 1; DescriptiveStatistics op1 = new DescriptiveStatistics(); DescriptiveStatistics op2 = new DescriptiveStatistics(); for (int k = 0; k < repeat; k++) { @@ -19,7 +19,7 @@ public class MyBasicOperationTest2 { Random r = new Random(); int low = 0; // inclusive int high = 256; // exclusive - int packNum = 128; + int packNum = 2; int packWidth = 8; // equal to one byte length byte[] buf = new byte[packNum]; for (int i = 0; i < packNum; i++) { @@ -32,7 +32,7 @@ public class MyBasicOperationTest2 { long start = System.nanoTime(); for (int i = 0; i < packNum; i++) { sum += BytesUtils.bytesToLong(buf, packWidth * i, packWidth); -// System.out.println(BytesUtils.bytesToLong(buf, packWidth * i, packWidth)); + System.out.println(BytesUtils.bytesToLong(buf, packWidth * i, packWidth)); } long elapsedTime = System.nanoTime() - start; System.out.println(elapsedTime / 1000.0 + "us"); @@ -47,7 +47,7 @@ public class MyBasicOperationTest2 { sum += (buf[i] & 0xff) << 8; // &0xff is to convert bytes to unsigned bytes // op2_b: 把一个byte的高位x个比特装到一个long的低位x个比特 // TODO 如何把一个byte一次分成高位x个比特和低位y个比特 -// System.out.println("---"); + System.out.println("---"); // System.out.println(buf[i]); // System.out.println((buf[i] & 0xff & 0b11100000) >> 5); sum += (buf[i] & 0xff & 0b11100000) >> 5; @@ -55,6 +55,14 @@ public class MyBasicOperationTest2 { // op2_c: 把一个byte的低位y个比特装到一个long的从低位到高位pos=packWidth-1的开始 // System.out.println((buf[i] & 0xff & ~0b11100000) << 3); sum += (buf[i] & 0xff & ~0b11100000) << 3; + + // op2_d: 把一个byte里的一部分比特装到一个long里 + int oneNum = 2; + int mask = (int) Math.pow(2, oneNum) - 1; + int shift = 4; + mask = mask << shift; // 0b00110000 + System.out.println(mask); + System.out.println((buf[i] & 0xff & mask) >> shift); } elapsedTime = System.nanoTime() - start; System.out.println(elapsedTime / 1000.0 + "us"); diff --git a/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest2.java b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java similarity index 66% copy from session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest2.java copy to session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java index eed2b00e6e..a281b1c117 100644 --- a/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest2.java +++ b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest3.java @@ -3,15 +3,19 @@ package org.apache.iotdb.session; import java.text.DecimalFormat; import java.util.Random; import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; +import org.apache.iotdb.tsfile.common.constant.TsFileConstant; import org.apache.iotdb.tsfile.utils.BytesUtils; +import org.junit.Assert; -public class MyBasicOperationTest2 { +public class MyBasicOperationTest3 { public static void main(String[] args) { // op1: long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i, packWidth); - // op2: put bytes as a whole into long + // op2: put bytes as a whole into long, i.e., BytesUtils.bytesToLong2(deltaBuf, packWidth * i, packWidth); int repeat = 1000000; + int packNum = 128; + int packWidth = 2; DescriptiveStatistics op1 = new DescriptiveStatistics(); DescriptiveStatistics op2 = new DescriptiveStatistics(); for (int k = 0; k < repeat; k++) { @@ -19,52 +23,51 @@ public class MyBasicOperationTest2 { Random r = new Random(); int low = 0; // inclusive int high = 256; // exclusive - int packNum = 128; - int packWidth = 8; // equal to one byte length - byte[] buf = new byte[packNum]; + byte[] buf = new byte[packNum * 8]; for (int i = 0; i < packNum; i++) { - int v = r.nextInt(high - low) + low; +// int v = r.nextInt(high - low) + low; + int v = 187; BytesUtils.longToBytes(v, buf, i * packWidth, packWidth); } // test op1 - long sum = 0; + long[] value1 = new long[packNum]; long start = System.nanoTime(); for (int i = 0; i < packNum; i++) { - sum += BytesUtils.bytesToLong(buf, packWidth * i, packWidth); + value1[i] = BytesUtils.bytesToLong(buf, packWidth * i, packWidth); // System.out.println(BytesUtils.bytesToLong(buf, packWidth * i, packWidth)); } long elapsedTime = System.nanoTime() - start; - System.out.println(elapsedTime / 1000.0 + "us"); - System.out.println(sum); +// System.out.println(elapsedTime / 1000.0 + "us"); +// System.out.println(sum); op1.addValue(elapsedTime / 1000.0); // test op2 - sum = 0; + long[] value2 = new long[packNum]; start = System.nanoTime(); for (int i = 0; i < packNum; i++) { - // op2_a: 把一个byte装到一个long里从低位到高位偏移offset，又即byte的最低位在long中的pos（从低到高从0开始数） - sum += (buf[i] & 0xff) << 8; // &0xff is to convert bytes to unsigned bytes - // op2_b: 把一个byte的高位x个比特装到一个long的低位x个比特 - // TODO 如何把一个byte一次分成高位x个比特和低位y个比特 -// System.out.println("---"); -// System.out.println(buf[i]); -// System.out.println((buf[i] & 0xff & 0b11100000) >> 5); - sum += (buf[i] & 0xff & 0b11100000) >> 5; - - // op2_c: 把一个byte的低位y个比特装到一个long的从低位到高位pos=packWidth-1的开始 -// System.out.println((buf[i] & 0xff & ~0b11100000) << 3); - sum += (buf[i] & 0xff & ~0b11100000) << 3; + value2[i] = BytesUtils.bytesToLong2(buf, packWidth * i, packWidth); +// System.out.println(BytesUtils.bytesToLong2(buf, packWidth * i, packWidth)); +// Assert.assertEquals(value1[i], value2[i]); } elapsedTime = System.nanoTime() - start; - System.out.println(elapsedTime / 1000.0 + "us"); - System.out.println(sum); +// System.out.println(elapsedTime / 1000.0 + "us"); +// System.out.println(sum2); op2.addValue(elapsedTime / 1000.0); + + System.out.println("---------------"); + for (int i = 0; i < packNum; i++) { + Assert.assertEquals(value1[i], value2[i]); + } } - printStat(op1, "op1-convertBitToLong"); - printStat(op2, "op2-compareByte"); + printStat(op1, "op1-bytesToLong"); + printStat(op2, "op2-bytesToLong2"); System.out.println("op1/op2=" + op1.getMean() / op2.getMean()); System.out.println("op2/op1=" + op2.getMean() / op1.getMean()); + System.out.println("repeat=" + repeat); + System.out.println("packNum=" + packNum); + System.out.println("packWidth=" + packWidth); + TsFileConstant.printByteToLongStatistics(); } private static String printStat(DescriptiveStatistics statistics, String name) { diff --git a/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java index c6fa88c7a7..6024cd118a 100644 --- a/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java +++ b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java @@ -53,29 +53,29 @@ public class MyRealDataTest1_WriteAndQuery { private static final String queryFormat_UDF = "select M4(%1$s,'tqs'='%3$d','tqe'='%4$d','w'='%5$d') from %2$s where time>=%3$d and time<%4$d"; - // private static String device = "root.game"; - // private static String measurement = "s6"; - // private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE; - // private static String timestamp_precision = "ns"; // ns, us, ms - // private static long dataMinTime = 0; - // private static long dataMaxTime = 617426057626L; - // private static long total_time_length = dataMaxTime - dataMinTime; - // private static int total_point_number = 1200000; - // private static int iotdb_chunk_point_size = 100000; - // private static long chunkAvgTimeLen = (long) Math - // .ceil(total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size)); - // private static String filePath = - // - // "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\BallSpeed.csv"; - // private static int deletePercentage = 0; // 0 means no deletes. 0-100 - // private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示 - // private static int timeIdx = 0; // 时间戳idx，从0开始 - // private static int valueIdx = 1; // 值idx，从0开始 - // private static int w = 2; - // private static long range = total_time_length; - // private static boolean enableRegularityTimeDecode = true; - // private static long regularTimeInterval = 511996L; - // private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV + private static String device = "root.game"; + private static String measurement = "s6"; + private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE; + private static String timestamp_precision = "ns"; // ns, us, ms + private static long dataMinTime = 0; + private static long dataMaxTime = 617426057626L; + private static long total_time_length = dataMaxTime - dataMinTime; + private static int total_point_number = 1200000; + private static int iotdb_chunk_point_size = 100000; + private static long chunkAvgTimeLen = (long) Math + .ceil(total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size)); + private static String filePath = + + "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\BallSpeed.csv"; + private static int deletePercentage = 0; // 0 means no deletes. 0-100 + private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示 + private static int timeIdx = 0; // 时间戳idx，从0开始 + private static int valueIdx = 1; // 值idx，从0开始 + private static int w = 2; + private static long range = total_time_length; + private static boolean enableRegularityTimeDecode = false; + private static long regularTimeInterval = 511996L; + private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV // private static String device = "root.debs2012"; // private static String measurement = "mf03"; @@ -101,30 +101,30 @@ public class MyRealDataTest1_WriteAndQuery { // private static long regularTimeInterval = 10000900L; // private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV - private static String device = "root.debs2012"; - private static String measurement = "mf03"; - private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE; - private static String timestamp_precision = "ns"; // ns, us, ms - private static long dataMinTime = 1329955200008000000L; - private static long dataMaxTime = 1329965999991000000L; - private static long total_time_length = dataMaxTime - dataMinTime; - private static int total_point_number = 1076102; - private static int iotdb_chunk_point_size = 100000; - private static long chunkAvgTimeLen = - (long) - Math.ceil( - total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size)); - private static String filePath = - "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\MF03_2.csv"; - private static int deletePercentage = 0; // 0 means no deletes. 0-100 - private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示 - private static int timeIdx = 0; // 时间戳idx，从0开始 - private static int valueIdx = 1; // 值idx，从0开始 - private static int w = 2; - private static long range = total_time_length; - private static boolean enableRegularityTimeDecode = true; - private static long regularTimeInterval = 10000000L; - private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV +// private static String device = "root.debs2012"; +// private static String measurement = "mf03"; +// private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE; +// private static String timestamp_precision = "ns"; // ns, us, ms +// private static long dataMinTime = 1329955200008000000L; +// private static long dataMaxTime = 1329965999991000000L; +// private static long total_time_length = dataMaxTime - dataMinTime; +// private static int total_point_number = 1076102; +// private static int iotdb_chunk_point_size = 100000; +// private static long chunkAvgTimeLen = +// (long) +// Math.ceil( +// total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size)); +// private static String filePath = +// "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\MF03_2.csv"; +// private static int deletePercentage = 0; // 0 means no deletes. 0-100 +// private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示 +// private static int timeIdx = 0; // 时间戳idx，从0开始 +// private static int valueIdx = 1; // 值idx，从0开始 +// private static int w = 2; +// private static long range = total_time_length; +// private static boolean enableRegularityTimeDecode = true; +// private static long regularTimeInterval = 10000000L; +// private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV // private static String device = "root.kobelco.trans.03.1090001603.2401604"; // private static String measurement = "KOB_0002_00_67"; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java index 75be680e75..bb9492fdb2 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java @@ -53,5 +53,29 @@ public class TsFileConstant { public static DescriptiveStatistics prepareAllRegulars = new DescriptiveStatistics(); - private TsFileConstant() {} + public static long bytesToLong_byteNum1_wholeByte = 0; + public static long bytesToLong_byteNum1_smallByte = 0; + public static long byteToLong_byteNums_firstByte_wholeByte = 0; + public static long byteToLong_byteNums_firstByte_smallByte = 0; + public static long byteToLong_byteNums_lastByte_wholeByte = 0; + public static long byteToLong_byteNums_lastByte_smallByte = 0; + public static long byteToLong_byteNums_middleWholeByte = 0; + + private TsFileConstant() { + } + + public static void printByteToLongStatistics() { + System.out.println("bytesToLong_byteNum1_wholeByte=" + bytesToLong_byteNum1_wholeByte); + System.out.println("bytesToLong_byteNum1_smallByte=" + bytesToLong_byteNum1_smallByte); + System.out.println( + "byteToLong_byteNums_firstByte_wholeByte=" + byteToLong_byteNums_firstByte_wholeByte); + System.out.println( + "byteToLong_byteNums_firstByte_smallByte=" + byteToLong_byteNums_firstByte_smallByte); + System.out.println( + "byteToLong_byteNums_lastByte_wholeByte=" + byteToLong_byteNums_lastByte_wholeByte); + System.out.println( + "byteToLong_byteNums_lastByte_smallByte=" + byteToLong_byteNums_lastByte_smallByte); + System.out + .println("byteToLong_byteNums_middleWholeByte=" + byteToLong_byteNums_middleWholeByte); + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java index 4f91f0b589..e799d33268 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java @@ -19,6 +19,10 @@ package org.apache.iotdb.tsfile.encoding.decoder; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; import org.apache.iotdb.tsfile.encoding.encoder.DeltaBinaryEncoder; @@ -27,15 +31,9 @@ import org.apache.iotdb.tsfile.utils.BytesUtils; import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Map; - /** * This class is a decoder for decoding the byte array that encoded by {@code - * DeltaBinaryEncoder}.DeltaBinaryDecoder just supports integer and long values.<br> - * . + * DeltaBinaryEncoder}.DeltaBinaryDecoder just supports integer and long values.<br> . * * @see DeltaBinaryEncoder */ @@ -44,16 +42,24 @@ public abstract class DeltaBinaryDecoder extends Decoder { protected long count = 0; protected byte[] deltaBuf; - /** the first value in one pack. */ + /** + * the first value in one pack. + */ protected int readIntTotalCount = 0; protected int nextReadIndex = 0; - /** max bit length of all value in a pack. */ + /** + * max bit length of all value in a pack. + */ protected int packWidth; - /** data number in this pack. */ + /** + * data number in this pack. + */ protected int packNum; - /** how many bytes data takes after encoding. */ + /** + * how many bytes data takes after encoding. + */ protected int encodingLength; public DeltaBinaryDecoder() { @@ -86,7 +92,9 @@ public abstract class DeltaBinaryDecoder extends Decoder { private int firstValue; private int[] data; private int previous; - /** minimum value for all difference. */ + /** + * minimum value for all difference. + */ private int minDeltaBase; public IntDeltaDecoder() { @@ -170,7 +178,9 @@ public abstract class DeltaBinaryDecoder extends Decoder { private long firstValue; private long[] data; private long previous; - /** minimum value for all difference. */ + /** + * minimum value for all difference. + */ private long minDeltaBase; private boolean enableRegularityTimeDecode; @@ -239,7 +249,7 @@ public abstract class DeltaBinaryDecoder extends Decoder { } } else if (newRegularDelta < 0 || newRegularDelta - >= Math.pow(2, packWidth)) { // no need to compare equality cause impossible + >= Math.pow(2, packWidth)) { // no need to compare equality cause impossible for (int i = 0; i < packNum; i++) { long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i, packWidth); data[i] = previous + minDeltaBase + v; @@ -267,6 +277,8 @@ public abstract class DeltaBinaryDecoder extends Decoder { if (newRegularDelta != 0) { // otherwise newRegularDelta=0 so leave byteArray as initial zeros + // TODO consider if these steps can be accelerated + // put bit-packed newRegularDelta starting at position i, // and pad the front and back with newRegularDeltas // 1. deal with padding the first byte @@ -345,7 +357,7 @@ public abstract class DeltaBinaryDecoder extends Decoder { // System.out.println("[RL]equals"); // TsFileConstant.countForRegularEqual++; } else { - long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i, packWidth); + long v = BytesUtils.bytesToLong2(deltaBuf, packWidth * i, packWidth); data[i] = previous + minDeltaBase + v; // System.out.println("[RL]no"); // TsFileConstant.countForRegularNOTEqual++; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java index 622b228731..49dee499e8 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java @@ -18,26 +18,24 @@ */ package org.apache.iotdb.tsfile.utils; -import org.apache.iotdb.tsfile.common.conf.TSFileConfig; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.io.InputStream; import java.util.List; +import org.apache.iotdb.tsfile.common.conf.TSFileConfig; +import org.apache.iotdb.tsfile.common.constant.TsFileConstant; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * BytesUtils is a utility class. It provide conversion among byte array and other type including - * integer, long, float, boolean, double and string. <br> - * It also provide other usable function as follow:<br> - * reading function which receives InputStream. <br> - * concat function to join a list of byte array to one.<br> - * get and set one bit in a byte array. + * integer, long, float, boolean, double and string. <br> It also provide other usable function as + * follow:<br> reading function which receives InputStream. <br> concat function to join a list of + * byte array to one.<br> get and set one bit in a byte array. */ public class BytesUtils { - private BytesUtils() {} + private BytesUtils() { + } private static final Logger LOG = LoggerFactory.getLogger(BytesUtils.class); @@ -48,19 +46,20 @@ public class BytesUtils { * @return byte[4] for integer */ public static byte[] intToBytes(int i) { - return new byte[] { - (byte) ((i >> 24) & 0xFF), - (byte) ((i >> 16) & 0xFF), - (byte) ((i >> 8) & 0xFF), - (byte) (i & 0xFF) + return new byte[]{ + (byte) ((i >> 24) & 0xFF), + (byte) ((i >> 16) & 0xFF), + (byte) ((i >> 8) & 0xFF), + (byte) (i & 0xFF) }; } /** - * integer convert to byte array, then write four bytes to parameter desc start from index:offset. + * integer convert to byte array, then write four bytes to parameter desc start from + * index:offset. * - * @param i integer to convert - * @param desc byte array be written + * @param i integer to convert + * @param desc byte array be written * @param offset position in desc byte array that conversion result should start * @return byte array */ @@ -81,8 +80,8 @@ public class BytesUtils { * * @param srcNum input integer variable * @param result byte array to convert - * @param pos start position - * @param width bit-width + * @param pos start position + * @param width bit-width */ public static void intToBytes(int srcNum, byte[] result, int pos, int width) { int temp = 0; @@ -160,7 +159,7 @@ public class BytesUtils { /** * convert four-bytes byte array cut from parameters to integer. * - * @param bytes source bytes which length should be greater than 4 + * @param bytes source bytes which length should be greater than 4 * @param offset position in parameter byte array that conversion result should start * @return integer */ @@ -182,8 +181,8 @@ public class BytesUtils { * given a byte array, read width bits from specified position bits and convert it to an integer. * * @param result input byte array - * @param pos bit offset rather than byte offset - * @param width bit-width + * @param pos bit offset rather than byte offset + * @param width bit-width * @return integer variable */ public static int bytesToInt(byte[] result, int pos, int width) { @@ -217,8 +216,8 @@ public class BytesUtils { /** * float convert to boolean, then write four bytes to parameter desc start from index:offset. * - * @param x float - * @param desc byte array be written + * @param x float + * @param desc byte array be written * @param offset position in desc byte array that conversion result should start */ public static void floatToBytes(float x, byte[] desc, int offset) { @@ -257,7 +256,7 @@ public class BytesUtils { /** * convert four-bytes byte array cut from parameters to float. * - * @param b source bytes which length should be greater than 4 + * @param b source bytes which length should be greater than 4 * @param offset position in parameter byte array that conversion result should start * @return float */ @@ -296,8 +295,8 @@ public class BytesUtils { /** * convert double to byte into the given byte array started from offset. * - * @param d input double - * @param bytes target byte[] + * @param d input double + * @param bytes target byte[] * @param offset start pos */ public static void doubleToBytes(double d, byte[] bytes, int offset) { @@ -340,7 +339,7 @@ public class BytesUtils { /** * convert eight-bytes byte array cut from parameters to double. * - * @param bytes source bytes which length should be greater than 8 + * @param bytes source bytes which length should be greater than 8 * @param offset position in parameter byte array that conversion result should start * @return double */ @@ -395,10 +394,11 @@ public class BytesUtils { } /** - * boolean convert to byte array, then write four bytes to parameter desc start from index:offset. + * boolean convert to byte array, then write four bytes to parameter desc start from + * index:offset. * - * @param x input boolean - * @param desc byte array be written + * @param x input boolean + * @param desc byte array be written * @param offset position in desc byte array that conversion result should start * @return byte[1] */ @@ -428,7 +428,7 @@ public class BytesUtils { /** * convert one-bytes byte array cut from parameters to boolean. * - * @param b source bytes which length should be greater than 1 + * @param b source bytes which length should be greater than 1 * @param offset position in parameter byte array that conversion result should start * @return boolean */ @@ -452,10 +452,9 @@ public class BytesUtils { /** * specify the result array length. then, convert long to Big-Endian byte from low to high. <br> - * e.g.<br> - * the binary presentation of long number 1000L is {6 bytes equal 0000000} 00000011 11101000<br> - * if len = 2, it will return byte array :{00000011 11101000}(Big-Endian) if len = 1, it will - * return byte array :{11101000}. + * e.g.<br> the binary presentation of long number 1000L is {6 bytes equal 0000000} 00000011 + * 11101000<br> if len = 2, it will return byte array :{00000011 11101000}(Big-Endian) if len = 1, + * it will return byte array :{11101000}. * * @param num long variable to be converted * @param len length of result byte array @@ -472,8 +471,8 @@ public class BytesUtils { /** * long convert to byte array, then write four bytes to parameter desc start from index:offset. * - * @param num input long variable - * @param desc byte array be written + * @param num input long variable + * @param desc byte array be written * @param offset position in desc byte array that conversion result should start * @return byte array */ @@ -491,8 +490,8 @@ public class BytesUtils { * * @param srcNum input long variable * @param result byte array to convert - * @param pos start position - * @param width bit-width + * @param pos start position + * @param width bit-width */ public static void longToBytes(long srcNum, byte[] result, int pos, int width) { int temp = 0; @@ -524,12 +523,11 @@ public class BytesUtils { /** * specify the input byte array length. then, convert byte array to long value from low to high. * <br> - * e.g.<br> - * the input byte array is {00000011 11101000}. if len = 2, return 1000 if len = 1, return - * 232(only calculate the low byte). + * e.g.<br> the input byte array is {00000011 11101000}. if len = 2, return 1000 if len = 1, + * return 232(only calculate the low byte). * * @param byteNum byte array to be converted - * @param len length of input byte array to be converted + * @param len length of input byte array to be converted * @return long */ public static long bytesToLong(byte[] byteNum, int len) { @@ -545,8 +543,8 @@ public class BytesUtils { * given a byte array, read width bits from specified pos bits and convert it to an long. * * @param result input byte array - * @param pos bit offset rather than byte offset - * @param width bit-width + * @param pos bit offset rather than byte offset + * @param width bit-width * @return long variable */ public static long bytesToLong(byte[] result, int pos, int width) { @@ -563,28 +561,85 @@ public class BytesUtils { * given a byte array, read width bits from specified pos bits and convert it to an long. * * @param result input byte array - * @param pos bit offset rather than byte offset - * @param width bit-width + * @param pos bit offset rather than byte offset + * @param width bit-width * @return long variable */ public static long bytesToLong2(byte[] result, int pos, int width) { - long value = 0; -// int temp = 0; -// for (int i = 0; i < width; i++) { -// temp = (pos + width - 1 - i) / 8; -// value = setLongN(value, i, getByteN(result[temp], pos + width - 1 - i)); -// } - //TODO new implementation + // pos is global over the byte array, from low to high, starting from 0 + // TODO new implementation + if (width == 0) { + return 0; + } + + int endPos = pos + width - 1; + int startByte = pos / 8; + int endByte = endPos / 8; + int byteNum = endByte - startByte + 1; + int startPosInByte = pos % 8; // in a byte from high to low bits starting from 0 + int endPosInByte = endPos % 8; + + // TODO if in the same byte + if (byteNum == 1) { + if (endPosInByte - startPosInByte == 7) { + // put the whole byte into the long value + TsFileConstant.bytesToLong_byteNum1_wholeByte++; + return result[startByte] & 0xff; + } else { + // put bits in the byte from the global position pos to pos+width-1 into the long value + TsFileConstant.bytesToLong_byteNum1_smallByte++; + int mask = (int) Math.pow(2, 8 - width) - 1; // TODO consider if this to make static + mask = (~mask & 0xff) >> startPosInByte; + return (result[startByte] & 0xff & mask) >> (7 - endPosInByte); + } + } + // TODO if across two bytes + else { + long value = 0; + // 1. deal with the first byte + int shift = width - (8 - startPosInByte); + if (startPosInByte == 0) { + // put the whole byte into the long value's front place among the last width bits + TsFileConstant.byteToLong_byteNums_firstByte_wholeByte++; + value = value | ((result[startByte] & 0xff) << shift); + } else { + // put the bits in the first byte from relative position pos%8 to the end into the long value's front place among the last width bits + TsFileConstant.byteToLong_byteNums_firstByte_smallByte++; + int mask = + (int) Math.pow(2, 8 - startPosInByte) - 1; // TODO consider if this to make static + value = value | ((result[startByte] & 0xff & mask) << shift); + } + + // 2. deal with the last byte + if (endPosInByte == 7) { + // put the whole byte into the long value's back place among the last width bits + TsFileConstant.byteToLong_byteNums_lastByte_wholeByte++; + value = value | (result[endByte] & 0xff); + } else { + // put the bits in the last byte from relative position 0 to (pos+width-1)%8 into the long value's back place among the last width bits + TsFileConstant.byteToLong_byteNums_lastByte_smallByte++; + int mask = + (int) Math.pow(2, 7 - endPosInByte) - 1; // TODO consider if this to make static + value = value | ((result[endByte] & 0xff & ~mask) >> (7 - endPosInByte)); + } + + // 3. deal with the middle bytes + for (int k = startByte + 1; k < endByte; k++) { + TsFileConstant.byteToLong_byteNums_middleWholeByte++; + shift -= 8; + value = value | ((result[k] & 0xff) << shift); + } + return value; + } - return value; } /** * convert eight-bytes byte array cut from parameters to long. * * @param byteNum source bytes which length should be greater than 8 - * @param len length of input byte array to be converted - * @param offset position in parameter byte array that conversion result should start + * @param len length of input byte array to be converted + * @param offset position in parameter byte array that conversion result should start * @return long */ public static long bytesToLongFromOffset(byte[] byteNum, int len, int offset) { @@ -658,8 +713,8 @@ public class BytesUtils { /** * cut out specified length byte array from parameter start from input byte array src and return. * - * @param src input byte array - * @param start start index of src + * @param src input byte array + * @param start start index of src * @param length cut off length * @return byte array */ @@ -676,12 +731,11 @@ public class BytesUtils { } /** - * get one bit in input integer. the offset is from low to high and start with 0<br> - * e.g.<br> + * get one bit in input integer. the offset is from low to high and start with 0<br> e.g.<br> * data:1000(00000000 00000000 00000011 11101000), if offset is 4, return 0(111 "0" 1000) if * offset is 9, return 1(00000 "1" 1 11101000). * - * @param data input int variable + * @param data input int variable * @param offset bit offset * @return 0 or 1 */ @@ -696,14 +750,13 @@ public class BytesUtils { /** * set one bit in input integer. the offset is from low to high and start with index 0<br> - * e.g.<br> - * data:1000({00000000 00000000 00000011 11101000}), if offset is 4, value is 1, return + * e.g.<br> data:1000({00000000 00000000 00000011 11101000}), if offset is 4, value is 1, return * 1016({00000000 00000000 00000011 111 "1" 1000}) if offset is 9, value is 0 return 488({00000000 * 00000000 000000 "0" 1 11101000}) if offset is 0, value is 0 return 1000(no change). * - * @param data input int variable + * @param data input int variable * @param offset bit offset - * @param value value to set + * @param value value to set * @return int variable */ public static int setIntN(int data, int offset, int value) { @@ -716,12 +769,11 @@ public class BytesUtils { } /** - * get one bit in input byte. the offset is from low to high and start with 0<br> - * e.g.<br> + * get one bit in input byte. the offset is from low to high and start with 0<br> e.g.<br> * data:16(00010000), if offset is 4, return 1(000 "1" 0000) if offset is 7, return 0("0" * 0010000). * - * @param data input byte variable + * @param data input byte variable * @param offset bit offset * @return 0/1 */ @@ -735,14 +787,13 @@ public class BytesUtils { } /** - * set one bit in input byte. the offset is from low to high and start with index 0<br> - * e.g.<br> + * set one bit in input byte. the offset is from low to high and start with index 0<br> e.g.<br> * data:16(00010000), if offset is 4, value is 0, return 0({000 "0" 0000}) if offset is 1, value * is 1, return 18({00010010}) if offset is 0, value is 0, return 16(no change). * - * @param data input byte variable + * @param data input byte variable * @param offset bit offset - * @param value value to set + * @param value value to set * @return byte variable */ public static byte setByteN(byte data, int offset, int value) { @@ -757,7 +808,7 @@ public class BytesUtils { /** * get one bit in input long. the offset is from low to high and start with 0. * - * @param data input long variable + * @param data input long variable * @param offset bit offset * @return 0/1 */ @@ -773,9 +824,9 @@ public class BytesUtils { /** * set one bit in input long. the offset is from low to high and start with index 0. * - * @param data input long variable + * @param data input long variable * @param offset bit offset - * @param value value to set + * @param value value to set * @return long variable */ public static long setLongN(long data, int offset, int value) { @@ -851,7 +902,7 @@ public class BytesUtils { * read bytes specified length from InputStream safely. * * @param count number of byte to read - * @param in InputStream + * @param in InputStream * @return byte array * @throws IOException cannot read from InputStream */
