Hi Experts, We are migrating Lucene from 2.4.1 to 8.11.2.
During Migration for a part of code, we are getting below exception in 8.11.2 based changes from Red line colored. ============= java.lang.IllegalArgumentException: TokenStream fields must be indexed and tokenized at org.apache.lucene.document.Field.<init>(Field.java:152) I tied few options. But, could not able to resolve the error. Beiiw Can somebody of you please help me to identify, where it is going as wrong? We had code based on 2.4.1 as like below: =================================== Int currentVal<http://10.238.236.101:8080/source/s?defs=currentVal&project=2025_RTM> = //some value PayloadTokenStream<http://10.238.236.101:8080/source/s?defs=PayloadTokenStream&project=2025_RTM> tokenStream<http://10.238.236.101:8080/source/s?refs=tokenStream&project=2025_RTM> = new PayloadTokenStream<http://10.238.236.101:8080/source/s?defs=PayloadTokenStream&project=2025_RTM>(); tokenStream<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/serviceplugins/src/com/sap/businessobjects/platform/search/lucene/index/engine/LuceneIndexEngine.java#tokenStream>.setPayload<http://10.238.236.101:8080/source/s?defs=setPayload&project=2025_RTM>(currentVal<http://10.238.236.101:8080/source/s?defs=currentVal&project=2025_RTM>); lucField<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/serviceplugins/src/com/sap/businessobjects/platform/search/lucene/index/engine/LuceneIndexEngine.java#lucField> = new Field<http://10.238.236.101:8080/source/s?defs=Field&project=2025_RTM>(config<http://10.238.236.101:8080/source/s?defs=config&project=2025_RTM>.payloadUid<http://10.238.236.101:8080/source/s?defs=payloadUid&project=2025_RTM>().name<http://10.238.236.101:8080/source/s?defs=name&project=2025_RTM>, tokenStream<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/serviceplugins/src/com/sap/businessobjects/platform/search/lucene/index/engine/LuceneIndexEngine.java#tokenStream>); doc<http://10.238.236.101:8080/source/s?defs=doc&project=2025_RTM>.add<http://10.238.236.101:8080/source/s?defs=add&project=2025_RTM>(lucField<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/serviceplugins/src/com/sap/businessobjects/platform/search/lucene/index/engine/LuceneIndexEngine.java#lucField>); ...... public class PayloadTokenStream<http://10.238.236.101:8080/source/s?refs=PayloadTokenStream&project=2025_RTM> extends TokenStream<http://10.238.236.101:8080/source/s?defs=TokenStream&project=2025_RTM>{ public static String<http://10.238.236.101:8080/source/s?defs=String&project=2025_RTM> UID_PAYLOAD_START_VAL<http://10.238.236.101:8080/source/s?refs=UID_PAYLOAD_START_VAL&project=2025_RTM> = "_UID_"; private Token<http://10.238.236.101:8080/source/s?defs=Token&project=2025_RTM> token<http://10.238.236.101:8080/source/s?refs=token&project=2025_RTM> = new Token<http://10.238.236.101:8080/source/s?defs=Token&project=2025_RTM>(UID_PAYLOAD_START_VAL<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#UID_PAYLOAD_START_VAL>,0,0); private byte[] buffer<http://10.238.236.101:8080/source/s?refs=buffer&project=2025_RTM> = new byte[4]; private boolean returnToken<http://10.238.236.101:8080/source/s?refs=returnToken&project=2025_RTM> = false; public void setPayload<http://10.238.236.101:8080/source/s?refs=setPayload&project=2025_RTM>(int uid<http://10.238.236.101:8080/source/s?refs=uid&project=2025_RTM>){ buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>[0] = (byte)uid<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#uid>; buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>[1] = (byte)(uid<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#uid>>>8); buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>[2] = (byte)(uid<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#uid>>>16); buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>[3] = (byte)(uid<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#uid>>>24); token<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#token>.setPayload<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#setPayload>(new Payload<http://10.238.236.101:8080/source/s?defs=Payload&project=2025_RTM>(buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>)); returnToken = true; } public Token<http://10.238.236.101:8080/source/s?defs=Token&project=2025_RTM> next<http://10.238.236.101:8080/source/s?refs=next&project=2025_RTM>() throws IOException<http://10.238.236.101:8080/source/s?defs=IOException&project=2025_RTM>{ if (returnToken<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#returnToken>){ returnToken<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#returnToken> = false; return token<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#token>; } else { return null<http://10.238.236.101:8080/source/s?defs=null&project=2025_RTM>; } } } We have made code based on 8.11.2 as like below: ========================================== PayloadTokenStream tokenStream = new PayloadTokenStream(); tokenStream.setPayload(currentVal); FieldType fieldType = new FieldType(); lucField = new Field(config.payloadUid().name, tokenStream, fieldType); doc.add(lucField); ---- public class PayloadTokenStream<http://10.238.236.101:8080/source/s?refs=PayloadTokenStream&project=2025_RTM> extends TokenStream<http://10.238.236.101:8080/source/s?defs=TokenStream&project=2025_RTM>{ public static String<http://10.238.236.101:8080/source/s?defs=String&project=2025_RTM> UID_PAYLOAD_START_VAL<http://10.238.236.101:8080/source/s?refs=UID_PAYLOAD_START_VAL&project=2025_RTM> = "_UID_"; private byte[] buffer<http://10.238.236.101:8080/source/s?refs=buffer&project=2025_RTM> = new byte[4]; private boolean returnToken<http://10.238.236.101:8080/source/s?refs=returnToken&project=2025_RTM> = false; public void setPayload<http://10.238.236.101:8080/source/s?refs=setPayload&project=2025_RTM>(int uid<http://10.238.236.101:8080/source/s?refs=uid&project=2025_RTM>){ buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>[0] = (byte)uid<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#uid>; buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>[1] = (byte)(uid<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#uid>>>8); buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>[2] = (byte)(uid<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#uid>>>16); buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>[3] = (byte)(uid<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#uid>>>24); PayloadAttributeImpl attributeImpl = new PayloadAttributeImpl(new BytesRef(buffer)); addAttributeImpl(attributeImpl); returnToken = true; } public boolean incrementToken() throws IOException { if (returnToken){ returnToken = false; return true; } else { return false; } } } Regards Rajib
