http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/JavaBinCodec.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/JavaBinCodec.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/JavaBinCodec.java new file mode 100644 index 0000000..687525c --- /dev/null +++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/JavaBinCodec.java @@ -0,0 +1,820 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.common.util; + +import org.apache.solr.common.EnumFieldValue; +import org.noggit.CharArr; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.SolrInputField; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.*; +import java.util.Map.Entry; +import java.nio.ByteBuffer; + +/** + * The class is designed to optimaly serialize/deserialize any supported types in Solr response. As we know there are only a limited type of + * items this class can do it with very minimal amount of payload and code. There are 15 known types and if there is an + * object in the object tree which does not fall into these types, It must be converted to one of these. Implement an + * ObjectResolver and pass it over It is expected that this class is used on both end of the pipes. The class has one + * read method and one write method for each of the datatypes + * <p> + * Note -- Never re-use an instance of this class for more than one marshal or unmarshall operation. Always create a new + * instance. + */ +public class JavaBinCodec { + + public static final byte + NULL = 0, + BOOL_TRUE = 1, + BOOL_FALSE = 2, + BYTE = 3, + SHORT = 4, + DOUBLE = 5, + INT = 6, + LONG = 7, + FLOAT = 8, + DATE = 9, + MAP = 10, + SOLRDOC = 11, + SOLRDOCLST = 12, + BYTEARR = 13, + ITERATOR = 14, + /** + * this is a special tag signals an end. No value is associated with it + */ + END = 15, + + SOLRINPUTDOC = 16, + SOLRINPUTDOC_CHILDS = 17, + ENUM_FIELD_VALUE = 18, + MAP_ENTRY = 19, + // types that combine tag + length (or other info) in a single byte + TAG_AND_LEN = (byte) (1 << 5), + STR = (byte) (1 << 5), + SINT = (byte) (2 << 5), + SLONG = (byte) (3 << 5), + ARR = (byte) (4 << 5), // + ORDERED_MAP = (byte) (5 << 5), // SimpleOrderedMap (a NamedList subclass, and more common) + NAMED_LST = (byte) (6 << 5), // NamedList + EXTERN_STRING = (byte) (7 << 5); + + + private static byte VERSION = 2; + private ObjectResolver resolver; + protected FastOutputStream daos; + + public JavaBinCodec() { + } + + public JavaBinCodec(ObjectResolver resolver) { + this.resolver = resolver; + } + + public void marshal(Object nl, OutputStream os) throws IOException { + init(FastOutputStream.wrap(os)); + try { + daos.writeByte(VERSION); + writeVal(nl); + } finally { + daos.flushBuffer(); + } + } + + /** expert: sets a new output stream */ + public void init(FastOutputStream os) { + daos = os; + } + + byte version; + + public Object unmarshal(InputStream is) throws IOException { + FastInputStream dis = FastInputStream.wrap(is); + version = dis.readByte(); + if (version != VERSION) { + throw new RuntimeException("Invalid version (expected " + VERSION + + ", but " + version + ") or the data in not in 'javabin' format"); + } + return readVal(dis); + } + + + public SimpleOrderedMap<Object> readOrderedMap(DataInputInputStream dis) throws IOException { + int sz = readSize(dis); + SimpleOrderedMap<Object> nl = new SimpleOrderedMap<>(); + for (int i = 0; i < sz; i++) { + String name = (String) readVal(dis); + Object val = readVal(dis); + nl.add(name, val); + } + return nl; + } + + public NamedList<Object> readNamedList(DataInputInputStream dis) throws IOException { + int sz = readSize(dis); + NamedList<Object> nl = new NamedList<>(); + for (int i = 0; i < sz; i++) { + String name = (String) readVal(dis); + Object val = readVal(dis); + nl.add(name, val); + } + return nl; + } + + public void writeNamedList(NamedList<?> nl) throws IOException { + writeTag(nl instanceof SimpleOrderedMap ? ORDERED_MAP : NAMED_LST, nl.size()); + for (int i = 0; i < nl.size(); i++) { + String name = nl.getName(i); + writeExternString(name); + Object val = nl.getVal(i); + writeVal(val); + } + } + + public void writeVal(Object val) throws IOException { + if (writeKnownType(val)) { + return; + } else { + Object tmpVal = val; + if (resolver != null) { + tmpVal = resolver.resolve(val, this); + if (tmpVal == null) return; // null means the resolver took care of it fully + if (writeKnownType(tmpVal)) return; + } + } + + writeVal(val.getClass().getName() + ':' + val.toString()); + } + + protected static final Object END_OBJ = new Object(); + + protected byte tagByte; + + public Object readVal(DataInputInputStream dis) throws IOException { + tagByte = dis.readByte(); + + // if ((tagByte & 0xe0) == 0) { + // if top 3 bits are clear, this is a normal tag + + // OK, try type + size in single byte + switch (tagByte >>> 5) { + case STR >>> 5: + return readStr(dis); + case SINT >>> 5: + return readSmallInt(dis); + case SLONG >>> 5: + return readSmallLong(dis); + case ARR >>> 5: + return readArray(dis); + case ORDERED_MAP >>> 5: + return readOrderedMap(dis); + case NAMED_LST >>> 5: + return readNamedList(dis); + case EXTERN_STRING >>> 5: + return readExternString(dis); + } + + switch (tagByte) { + case NULL: + return null; + case DATE: + return new Date(dis.readLong()); + case INT: + return dis.readInt(); + case BOOL_TRUE: + return Boolean.TRUE; + case BOOL_FALSE: + return Boolean.FALSE; + case FLOAT: + return dis.readFloat(); + case DOUBLE: + return dis.readDouble(); + case LONG: + return dis.readLong(); + case BYTE: + return dis.readByte(); + case SHORT: + return dis.readShort(); + case MAP: + return readMap(dis); + case SOLRDOC: + return readSolrDocument(dis); + case SOLRDOCLST: + return readSolrDocumentList(dis); + case BYTEARR: + return readByteArray(dis); + case ITERATOR: + return readIterator(dis); + case END: + return END_OBJ; + case SOLRINPUTDOC: + return readSolrInputDocument(dis); + case ENUM_FIELD_VALUE: + return readEnumFieldValue(dis); + case MAP_ENTRY: + return readMapEntry(dis); + } + + throw new RuntimeException("Unknown type " + tagByte); + } + + public boolean writeKnownType(Object val) throws IOException { + if (writePrimitive(val)) return true; + if (val instanceof NamedList) { + writeNamedList((NamedList<?>) val); + return true; + } + if (val instanceof SolrDocumentList) { // SolrDocumentList is a List, so must come before List check + writeSolrDocumentList((SolrDocumentList) val); + return true; + } + if (val instanceof Collection) { + writeArray((Collection) val); + return true; + } + if (val instanceof Object[]) { + writeArray((Object[]) val); + return true; + } + if (val instanceof SolrDocument) { + //this needs special treatment to know which fields are to be written + if (resolver == null) { + writeSolrDocument((SolrDocument) val); + } else { + Object retVal = resolver.resolve(val, this); + if (retVal != null) { + if (retVal instanceof SolrDocument) { + writeSolrDocument((SolrDocument) retVal); + } else { + writeVal(retVal); + } + } + } + return true; + } + if (val instanceof SolrInputDocument) { + writeSolrInputDocument((SolrInputDocument)val); + return true; + } + if (val instanceof Map) { + writeMap((Map) val); + return true; + } + if (val instanceof Iterator) { + writeIterator((Iterator) val); + return true; + } + if (val instanceof Iterable) { + writeIterator(((Iterable) val).iterator()); + return true; + } + if (val instanceof EnumFieldValue) { + writeEnumFieldValue((EnumFieldValue) val); + return true; + } + if (val instanceof Map.Entry) { + writeMapEntry((Map.Entry)val); + return true; + } + return false; + } + + public void writeTag(byte tag) throws IOException { + daos.writeByte(tag); + } + + public void writeTag(byte tag, int size) throws IOException { + if ((tag & 0xe0) != 0) { + if (size < 0x1f) { + daos.writeByte(tag | size); + } else { + daos.writeByte(tag | 0x1f); + writeVInt(size - 0x1f, daos); + } + } else { + daos.writeByte(tag); + writeVInt(size, daos); + } + } + + public void writeByteArray(byte[] arr, int offset, int len) throws IOException { + writeTag(BYTEARR, len); + daos.write(arr, offset, len); + } + + public byte[] readByteArray(DataInputInputStream dis) throws IOException { + byte[] arr = new byte[readVInt(dis)]; + dis.readFully(arr); + return arr; + } + + public void writeSolrDocument(SolrDocument doc) throws IOException { + List<SolrDocument> children = doc.getChildDocuments(); + int sz = doc.size() + (children==null ? 0 : children.size()); + writeTag(SOLRDOC); + writeTag(ORDERED_MAP, sz); + for (Map.Entry<String, Object> entry : doc) { + String name = entry.getKey(); + writeExternString(name); + Object val = entry.getValue(); + writeVal(val); + } + if (children != null) { + for (SolrDocument child : children) { + writeSolrDocument(child); + } + } + } + + public SolrDocument readSolrDocument(DataInputInputStream dis) throws IOException { + tagByte = dis.readByte(); + int size = readSize(dis); + SolrDocument doc = new SolrDocument(); + for (int i = 0; i < size; i++) { + String fieldName; + Object obj = readVal(dis); // could be a field name, or a child document + if (obj instanceof SolrDocument) { + doc.addChildDocument((SolrDocument)obj); + continue; + } else { + fieldName = (String)obj; + } + Object fieldVal = readVal(dis); + doc.setField(fieldName, fieldVal); + } + return doc; + } + + public SolrDocumentList readSolrDocumentList(DataInputInputStream dis) throws IOException { + SolrDocumentList solrDocs = new SolrDocumentList(); + List list = (List) readVal(dis); + solrDocs.setNumFound((Long) list.get(0)); + solrDocs.setStart((Long) list.get(1)); + solrDocs.setMaxScore((Float) list.get(2)); + + @SuppressWarnings("unchecked") + List<SolrDocument> l = (List<SolrDocument>) readVal(dis); + solrDocs.addAll(l); + return solrDocs; + } + + public void writeSolrDocumentList(SolrDocumentList docs) + throws IOException { + writeTag(SOLRDOCLST); + List<Number> l = new ArrayList<>(3); + l.add(docs.getNumFound()); + l.add(docs.getStart()); + l.add(docs.getMaxScore()); + writeArray(l); + writeArray(docs); + } + + public SolrInputDocument readSolrInputDocument(DataInputInputStream dis) throws IOException { + int sz = readVInt(dis); + float docBoost = (Float)readVal(dis); + SolrInputDocument sdoc = new SolrInputDocument(); + sdoc.setDocumentBoost(docBoost); + for (int i = 0; i < sz; i++) { + float boost = 1.0f; + String fieldName; + Object obj = readVal(dis); // could be a boost, a field name, or a child document + if (obj instanceof Float) { + boost = (Float)obj; + fieldName = (String)readVal(dis); + } else if (obj instanceof SolrInputDocument) { + sdoc.addChildDocument((SolrInputDocument)obj); + continue; + } else { + fieldName = (String)obj; + } + Object fieldVal = readVal(dis); + sdoc.setField(fieldName, fieldVal, boost); + } + return sdoc; + } + + public void writeSolrInputDocument(SolrInputDocument sdoc) throws IOException { + List<SolrInputDocument> children = sdoc.getChildDocuments(); + int sz = sdoc.size() + (children==null ? 0 : children.size()); + writeTag(SOLRINPUTDOC, sz); + writeFloat(sdoc.getDocumentBoost()); + for (SolrInputField inputField : sdoc.values()) { + if (inputField.getBoost() != 1.0f) { + writeFloat(inputField.getBoost()); + } + writeExternString(inputField.getName()); + writeVal(inputField.getValue()); + } + if (children != null) { + for (SolrInputDocument child : children) { + writeSolrInputDocument(child); + } + } + } + + + public Map<Object,Object> readMap(DataInputInputStream dis) + throws IOException { + int sz = readVInt(dis); + Map<Object,Object> m = new LinkedHashMap<>(); + for (int i = 0; i < sz; i++) { + Object key = readVal(dis); + Object val = readVal(dis); + m.put(key, val); + + } + return m; + } + + public void writeIterator(Iterator iter) throws IOException { + writeTag(ITERATOR); + while (iter.hasNext()) { + writeVal(iter.next()); + } + writeVal(END_OBJ); + } + + public List<Object> readIterator(DataInputInputStream fis) throws IOException { + ArrayList<Object> l = new ArrayList<>(); + while (true) { + Object o = readVal(fis); + if (o == END_OBJ) break; + l.add(o); + } + return l; + } + + public void writeArray(List l) throws IOException { + writeTag(ARR, l.size()); + for (int i = 0; i < l.size(); i++) { + writeVal(l.get(i)); + } + } + + public void writeArray(Collection coll) throws IOException { + writeTag(ARR, coll.size()); + for (Object o : coll) { + writeVal(o); + } + + } + + public void writeArray(Object[] arr) throws IOException { + writeTag(ARR, arr.length); + for (int i = 0; i < arr.length; i++) { + Object o = arr[i]; + writeVal(o); + } + } + + public List<Object> readArray(DataInputInputStream dis) throws IOException { + int sz = readSize(dis); + ArrayList<Object> l = new ArrayList<>(sz); + for (int i = 0; i < sz; i++) { + l.add(readVal(dis)); + } + return l; + } + + /** + * write {@link EnumFieldValue} as tag+int value+string value + * @param enumFieldValue to write + */ + public void writeEnumFieldValue(EnumFieldValue enumFieldValue) throws IOException { + writeTag(ENUM_FIELD_VALUE); + writeInt(enumFieldValue.toInt()); + writeStr(enumFieldValue.toString()); + } + + public void writeMapEntry(Entry<Object,Object> val) throws IOException { + writeTag(MAP_ENTRY); + writeVal(val.getKey()); + writeVal(val.getValue()); + } + + /** + * read {@link EnumFieldValue} (int+string) from input stream + * @param dis data input stream + * @return {@link EnumFieldValue} + */ + public EnumFieldValue readEnumFieldValue(DataInputInputStream dis) throws IOException { + Integer intValue = (Integer) readVal(dis); + String stringValue = (String) readVal(dis); + return new EnumFieldValue(intValue, stringValue); + } + + + public Map.Entry<Object,Object> readMapEntry(DataInputInputStream dis) throws IOException { + final Object key = readVal(dis); + final Object value = readVal(dis); + return new Map.Entry<Object,Object>() { + + @Override + public Object getKey() { + return key; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public String toString() { + return "MapEntry[" + key.toString() + ":" + value.toString() + "]"; + } + + @Override + public Object setValue(Object value) { + throw new UnsupportedOperationException(); + } + + @Override + public int hashCode() { + int result = 31; + result *=31 + getKey().hashCode(); + result *=31 + getValue().hashCode(); + return result; + } + + @Override + public boolean equals(Object obj) { + if(this == obj) { + return true; + } + if(!(obj instanceof Entry)) { + return false; + } + Map.Entry<Object, Object> entry = (Entry<Object, Object>) obj; + return (this.getKey().equals(entry.getKey()) && this.getValue().equals(entry.getValue())); + } + }; + } + + /** + * write the string as tag+length, with length being the number of UTF-8 bytes + */ + public void writeStr(String s) throws IOException { + if (s == null) { + writeTag(NULL); + return; + } + int end = s.length(); + int maxSize = end * 4; + if (bytes == null || bytes.length < maxSize) bytes = new byte[maxSize]; + int sz = ByteUtils.UTF16toUTF8(s, 0, end, bytes, 0); + + writeTag(STR, sz); + daos.write(bytes, 0, sz); + } + + byte[] bytes; + CharArr arr = new CharArr(); + + public String readStr(DataInputInputStream dis) throws IOException { + int sz = readSize(dis); + if (bytes == null || bytes.length < sz) bytes = new byte[sz]; + dis.readFully(bytes, 0, sz); + + arr.reset(); + ByteUtils.UTF8toUTF16(bytes, 0, sz, arr); + return arr.toString(); + } + + public void writeInt(int val) throws IOException { + if (val > 0) { + int b = SINT | (val & 0x0f); + + if (val >= 0x0f) { + b |= 0x10; + daos.writeByte(b); + writeVInt(val >>> 4, daos); + } else { + daos.writeByte(b); + } + + } else { + daos.writeByte(INT); + daos.writeInt(val); + } + } + + public int readSmallInt(DataInputInputStream dis) throws IOException { + int v = tagByte & 0x0F; + if ((tagByte & 0x10) != 0) + v = (readVInt(dis) << 4) | v; + return v; + } + + + public void writeLong(long val) throws IOException { + if ((val & 0xff00000000000000L) == 0) { + int b = SLONG | ((int) val & 0x0f); + if (val >= 0x0f) { + b |= 0x10; + daos.writeByte(b); + writeVLong(val >>> 4, daos); + } else { + daos.writeByte(b); + } + } else { + daos.writeByte(LONG); + daos.writeLong(val); + } + } + + public long readSmallLong(DataInputInputStream dis) throws IOException { + long v = tagByte & 0x0F; + if ((tagByte & 0x10) != 0) + v = (readVLong(dis) << 4) | v; + return v; + } + + public void writeFloat(float val) throws IOException { + daos.writeByte(FLOAT); + daos.writeFloat(val); + } + + public boolean writePrimitive(Object val) throws IOException { + if (val == null) { + daos.writeByte(NULL); + return true; + } else if (val instanceof String) { + writeStr((String) val); + return true; + } else if (val instanceof Number) { + + if (val instanceof Integer) { + writeInt(((Integer) val).intValue()); + return true; + } else if (val instanceof Long) { + writeLong(((Long) val).longValue()); + return true; + } else if (val instanceof Float) { + writeFloat(((Float) val).floatValue()); + return true; + } else if (val instanceof Double) { + daos.writeByte(DOUBLE); + daos.writeDouble(((Double) val).doubleValue()); + return true; + } else if (val instanceof Byte) { + daos.writeByte(BYTE); + daos.writeByte(((Byte) val).intValue()); + return true; + } else if (val instanceof Short) { + daos.writeByte(SHORT); + daos.writeShort(((Short) val).intValue()); + return true; + } + return false; + + } else if (val instanceof Date) { + daos.writeByte(DATE); + daos.writeLong(((Date) val).getTime()); + return true; + } else if (val instanceof Boolean) { + if ((Boolean) val) daos.writeByte(BOOL_TRUE); + else daos.writeByte(BOOL_FALSE); + return true; + } else if (val instanceof byte[]) { + writeByteArray((byte[]) val, 0, ((byte[]) val).length); + return true; + } else if (val instanceof ByteBuffer) { + ByteBuffer buf = (ByteBuffer) val; + writeByteArray(buf.array(),buf.position(),buf.limit() - buf.position()); + return true; + } else if (val == END_OBJ) { + writeTag(END); + return true; + } + return false; + } + + + public void writeMap(Map<?,?> val) throws IOException { + writeTag(MAP, val.size()); + for (Map.Entry<?,?> entry : val.entrySet()) { + Object key = entry.getKey(); + if (key instanceof String) { + writeExternString((String) key); + } else { + writeVal(key); + } + writeVal(entry.getValue()); + } + } + + + public int readSize(DataInputInputStream in) throws IOException { + int sz = tagByte & 0x1f; + if (sz == 0x1f) sz += readVInt(in); + return sz; + } + + + /** + * Special method for variable length int (copied from lucene). Usually used for writing the length of a + * collection/array/map In most of the cases the length can be represented in one byte (length < 127) so it saves 3 + * bytes/object + * + * @throws IOException If there is a low-level I/O error. + */ + public static void writeVInt(int i, FastOutputStream out) throws IOException { + while ((i & ~0x7F) != 0) { + out.writeByte((byte) ((i & 0x7f) | 0x80)); + i >>>= 7; + } + out.writeByte((byte) i); + } + + /** + * The counterpart for {@link #writeVInt(int, FastOutputStream)} + * + * @throws IOException If there is a low-level I/O error. + */ + public static int readVInt(DataInputInputStream in) throws IOException { + byte b = in.readByte(); + int i = b & 0x7F; + for (int shift = 7; (b & 0x80) != 0; shift += 7) { + b = in.readByte(); + i |= (b & 0x7F) << shift; + } + return i; + } + + + public static void writeVLong(long i, FastOutputStream out) throws IOException { + while ((i & ~0x7F) != 0) { + out.writeByte((byte) ((i & 0x7f) | 0x80)); + i >>>= 7; + } + out.writeByte((byte) i); + } + + public static long readVLong(DataInputInputStream in) throws IOException { + byte b = in.readByte(); + long i = b & 0x7F; + for (int shift = 7; (b & 0x80) != 0; shift += 7) { + b = in.readByte(); + i |= (long) (b & 0x7F) << shift; + } + return i; + } + + private int stringsCount = 0; + private Map<String, Integer> stringsMap; + private List<String> stringsList; + + public void writeExternString(String s) throws IOException { + if (s == null) { + writeTag(NULL); + return; + } + Integer idx = stringsMap == null ? null : stringsMap.get(s); + if (idx == null) idx = 0; + writeTag(EXTERN_STRING, idx); + if (idx == 0) { + writeStr(s); + if (stringsMap == null) stringsMap = new HashMap<>(); + stringsMap.put(s, ++stringsCount); + } + + } + + public String readExternString(DataInputInputStream fis) throws IOException { + int idx = readSize(fis); + if (idx != 0) {// idx != 0 is the index of the extern string + return stringsList.get(idx - 1); + } else {// idx == 0 means it has a string value + String s = (String) readVal(fis); + if (stringsList == null) stringsList = new ArrayList<>(); + stringsList.add(s); + return s; + } + } + + + public static interface ObjectResolver { + public Object resolve(Object o, JavaBinCodec codec) throws IOException; + } + + +}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/JsonRecordReader.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/JsonRecordReader.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/JsonRecordReader.java new file mode 100644 index 0000000..c3afde7 --- /dev/null +++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/JsonRecordReader.java @@ -0,0 +1,586 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.common.util; + + +import java.io.IOException; +import java.io.Reader; +import java.util.*; + +import org.noggit.JSONParser; + +import static org.noggit.JSONParser.*; + +/** + * A Streaming parser for json to emit one record at a time. + */ + +public class JsonRecordReader { + public static final String DELIM = "."; + + private Node rootNode = new Node("/", (Node) null); + + public static JsonRecordReader getInst(String split, List<String> fieldMappings) { + + JsonRecordReader jsonRecordReader = new JsonRecordReader(split); + for (String s : fieldMappings) { + String path = s; + int idx = s.indexOf(':'); + String fieldName = null; + if (idx > 0) { + fieldName = s.substring(0, idx); + path = s.substring(idx + 1); + } + jsonRecordReader.addField(path, fieldName, true, false); + } + return jsonRecordReader; + } + + /** + * A constructor called with a '|' separated list of path expressions + * which define sub sections of the JSON stream that are to be emitted as + * separate records. + * + * @param splitPath The PATH for which a record is emitted. Once the + * path tag is encountered, the Node.getInst method starts collecting wanted + * fields and at the close of the tag, a record is emitted containing all + * fields collected since the tag start. Once + * emitted the collected fields are cleared. Any fields collected in the + * parent tag or above will also be included in the record, but these are + * not cleared after emitting the record. + * <p> + * It uses the ' | ' syntax of PATH to pass in multiple paths. + */ + private JsonRecordReader(String splitPath) { + String[] splits = splitPath.split("\\|"); + for (String split : splits) { + split = split.trim(); + if (split.startsWith("//")) + throw new RuntimeException("split cannot start with '//': " + split); + if (split.length() == 0) + continue; + // The created Node has a name set to the full split attribute path + addField(split, split, false, true); + } + } + + /** + * Splits the path into a List of segments and calls build() to + * construct a tree of Nodes representing path segments. The resulting + * tree structure ends up describing all the paths we are interested in. + * + * @param path The path expression for this field + * @param fieldName The name for this field in the emitted record + * @param multiValued If 'true' then the emitted record will have values in + * a List<String> + * @param isRecord Flags that this PATH is from a forEach statement + */ + private void addField(String path, String fieldName, boolean multiValued, boolean isRecord) { + if (!path.startsWith("/")) throw new RuntimeException("All paths must start with '/' " + path); + List<String> paths = splitEscapeQuote(path); + if (paths.size() == 0) { + if (isRecord) rootNode.isRecord = true; + return;//the patrh is "/" + } + // deal with how split behaves when seperator starts a string! + if ("".equals(paths.get(0).trim())) + paths.remove(0); + rootNode.build(paths, fieldName, multiValued, isRecord, path); + rootNode.buildOptimize(); + } + + /** + * Uses {@link #streamRecords streamRecords} to getInst the JSON source but with + * a handler that collects all the emitted records into a single List which + * is returned upon completion. + * + * @param r the stream reader + * @return results a List of emitted records + */ + public List<Map<String, Object>> getAllRecords(Reader r) throws IOException { + final List<Map<String, Object>> results = new ArrayList<>(); + streamRecords(r, new Handler() { + @Override + public void handle(Map<String, Object> record, String path) { + results.add(record); + } + }); + return results; + } + + /** + * Creates an JSONParser on top of whatever reader has been + * configured. Then calls getInst() with a handler which is + * invoked forEach record emitted. + * + * @param r the stream reader + * @param handler The callback instance + */ + public void streamRecords(Reader r, Handler handler) throws IOException { + streamRecords(new JSONParser(r), handler); + } + + public void streamRecords(JSONParser parser, Handler handler) throws IOException { + rootNode.parse(parser, handler, + new LinkedHashMap<String, Object>(), + new Stack<Set<String>>(), false); + } + + + /** + * For each node/leaf in the Node tree there is one object of this class. + * This tree of objects represents all the Paths we are interested in. + * For each path segment of interest we create a node. In most cases the + * node (branch) is rather basic , but for the final portion (leaf) of any + * path we add more information to the Node. When parsing the JSON document + * we step though this tree as we stream records from the reader. If the JSON + * document departs from this tree we skip start tags till we are back on + * the tree. + */ + private static class Node { + String name; // generally: segment of the path represented by this Node + String fieldName; // the fieldname in the emitted record (key of the map) + String splitPath; // the full path from the forEach entity attribute + final LinkedHashMap<String, Node> childNodes = new LinkedHashMap<>(); // List of immediate child Nodes of this node + Node parent; // parent Node in the tree + boolean isLeaf = false; // flag: store/emit streamed text for this node + boolean isRecord = false; //flag: this Node starts a new record + Node wildCardChild; + Node recursiveWildCardChild; + private boolean useFqn = false; + + + public Node(String name, Node p) { + // Create a basic Node, suitable for the mid portions of any path. + // Node.pathName and Node.name are set to same value. + this.name = name; + parent = p; + } + + public Node(String name, String fieldName) { + // This is only called from build() when describing an attribute. + this.name = name; // a segment from the path + this.fieldName = fieldName; // name to store collected values against + } + + + /** + * Walk the Node tree propagating any wildDescentant information to + * child nodes. + */ + private void buildOptimize() { + if (parent != null && parent.recursiveWildCardChild != null && this.recursiveWildCardChild == null) { + this.recursiveWildCardChild = parent.recursiveWildCardChild; + } + for (Node n : childNodes.values()) n.buildOptimize(); + } + static final String WILDCARD_PATH = "*"; + static final String RECURSIVE_WILDCARD_PATH = "**"; + + /** + * Build a Node tree structure representing all paths of intrest to us. + * This must be done before parsing of the JSON stream starts. Each node + * holds one portion of an path. Taking each path segment in turn this + * method walks the Node tree and finds where the new segment should be + * inserted. It creates a Node representing a field's name, PATH and + * some flags and inserts the Node into the Node tree. + */ + private void build( + List<String> paths, // a List of segments from the split paths + String fieldName, // the fieldName assoc with this path + boolean multiValued, // flag if this fieldName is multiValued or not + boolean record, // is this path a record or a field + String path) { + // recursively walk the paths Lists adding new Nodes as required + String segment = paths.remove(0); // shift out next path segment + + if (segment.length() < 1) throw new RuntimeException("all pieces in path must be non empty " + path); + + // does this "name" already exist as a child node. + Node n = getOrAddNode(segment, childNodes); + if (paths.isEmpty()) { + // We have emptied paths, we are for the moment a leaf of the tree. + // When parsing the actual input we have traversed to a position + // where we actutally have to do something. getOrAddNode() will + // have created and returned a new minimal Node with name and + // pathName already populated. We need to add more information. + if (record) { + //wild cards cannot be used in split + assert !WILDCARD_PATH.equals(n.name); + assert !RECURSIVE_WILDCARD_PATH.equals(n.name); + // split attribute + n.isRecord = true; // flag: split attribute, prepare to emit rec + n.splitPath = fieldName; // the full split attribute path + } else { + if (n.name.equals(WILDCARD_PATH)) { + wildCardChild = n; + } + if (n.name.equals(RECURSIVE_WILDCARD_PATH)) { + recursiveWildCardChild = n.recursiveWildCardChild = n; + } + + // path with content we want to store and return + n.isLeaf = true; // we have to store text found here + n.fieldName = fieldName; // name to store collected text against + if ("$FQN".equals(n.fieldName)) { + n.fieldName = null; + n.useFqn = true; + } + } + } else { + //wildcards must only come at the end + if (WILDCARD_PATH.equals(name) || RECURSIVE_WILDCARD_PATH.equals(name)) + throw new RuntimeException("wild cards are allowed only in the end " + path); + // recurse to handle next paths segment + n.build(paths, fieldName, multiValued, record, path); + } + } + + private Node getOrAddNode(String pathName, Map<String, Node> children) { + Node n = children.get(pathName); + if (n != null) return n; + // new territory! add a new node for this path bitty + children.put(pathName, n = new Node(pathName, this)); + return n; + } + + /** + * Copies a supplied Map to a new Map which is returned. Used to copy a + * records values. If a fields value is a List then they have to be + * deep-copied for thread safety + */ + private static Map<String, Object> getDeepCopy(Map<String, Object> values) { + Map<String, Object> result = new LinkedHashMap<>(); + for (Map.Entry<String, Object> entry : values.entrySet()) { + if (entry.getValue() instanceof List) { + result.put(entry.getKey(), new ArrayList((List) entry.getValue())); + } else { + result.put(entry.getKey(), entry.getValue()); + } + } + return result; + } + + private void parse(JSONParser parser, + Handler handler, + Map<String, Object> values, + Stack<Set<String>> stack, // lists of values to purge + boolean recordStarted) throws IOException { + + int event = -1; + for (; ; ) { + event = parser.nextEvent(); + if (event == EOF) break; + if (event == OBJECT_START) { + handleObjectStart(parser, new HashSet<Node>(), handler, values, stack, recordStarted, null); + } else if (event == ARRAY_START) { + for (; ; ) { + event = parser.nextEvent(); + if (event == ARRAY_END) break; + if (event == OBJECT_START) { + handleObjectStart(parser, new HashSet<Node>(), handler, values, stack, recordStarted, null); + } + } + } + } + + } + + /** + * If a new tag is encountered, check if it is of interest or not by seeing + * if it matches against our node tree. If we have deperted from the node + * tree then walk back though the tree's ancestor nodes checking to see if + * any // expressions exist for the node and compare them against the new + * tag. If matched then "jump" to that node, otherwise ignore the tag. + * <p> + * Note, the list of // expressions found while walking back up the tree + * is chached in the HashMap decends. Then if the new tag is to be skipped, + * any inner chil tags are compared against the cache and jumped to if + * matched. + */ + private void handleObjectStart(final JSONParser parser, final Set<Node> childrenFound, + final Handler handler, final Map<String, Object> values, + final Stack<Set<String>> stack, boolean recordStarted, + MethodFrameWrapper frameWrapper) + throws IOException { + + final boolean isRecordStarted = recordStarted || isRecord; + Set<String> valuesAddedinThisFrame = null; + if (isRecord) { + // This Node is a match for an PATH from a forEach attribute, + // prepare for the clean up that will occurr when the record + // is emitted after its END_ELEMENT is matched + valuesAddedinThisFrame = new HashSet<>(); + stack.push(valuesAddedinThisFrame); + } else if (recordStarted) { + // This node is a child of some parent which matched against forEach + // attribute. Continue to add values to an existing record. + valuesAddedinThisFrame = stack.peek(); + } + + class Wrapper extends MethodFrameWrapper { + Wrapper(Node node, MethodFrameWrapper parent, String name) { + this.node = node; + this.parent = parent; + this.name = name; + } + + @Override + public void walk(int event) throws IOException { + if (event == OBJECT_START) { + node.handleObjectStart(parser, childrenFound, handler, values, stack, isRecordStarted, this); + } else if (event == ARRAY_START) { + for (; ; ) { + event = parser.nextEvent(); + if (event == ARRAY_END) break; + if (event == OBJECT_START) { + node.handleObjectStart(parser, childrenFound, handler, values, stack, isRecordStarted, this); + } + } + } + + } + } + + try { + for (; ; ) { + int event = parser.nextEvent(); + if (event == OBJECT_END) { + if (isRecord()) { + handler.handle(getDeepCopy(values), splitPath); + } + return; + } + assert event == STRING; + assert parser.wasKey(); + String name = parser.getString(); + + Node node = childNodes.get(name); + if (node == null) node = wildCardChild; + if (node == null) node = recursiveWildCardChild; + + if (node != null) { + if (node.isLeaf) {//this is a leaf collect data here + event = parser.nextEvent(); + String nameInRecord = node.fieldName == null ? getNameInRecord(name, frameWrapper, node) : node.fieldName; + MethodFrameWrapper runnable = null; + if (event == OBJECT_START || event == ARRAY_START) { + if (node.recursiveWildCardChild != null) runnable = new Wrapper(node, frameWrapper, name); + } + Object val = parseSingleFieldValue(event, parser, runnable); + if (val != null) { + putValue(values, nameInRecord, val); + if (isRecordStarted) valuesAddedinThisFrame.add(nameInRecord); + } + + } else { + event = parser.nextEvent(); + new Wrapper(node, frameWrapper, name).walk(event); + } + } else { + //this is not something we are interested in . skip it + event = parser.nextEvent(); + if (event == STRING || + event == LONG || + event == NUMBER || + event == BIGNUMBER || + event == BOOLEAN || + event == NULL) { + continue; + } + if (event == ARRAY_START) { + consumeTillMatchingEnd(parser, 0, 1); + continue; + } + if (event == OBJECT_START) { + consumeTillMatchingEnd(parser, 1, 0); + continue; + } else throw new RuntimeException("unexpected token " + event); + + } + } + } finally { + if ((isRecord() || !isRecordStarted) && !stack.empty()) { + Set<String> cleanThis = stack.pop(); + if (cleanThis != null) { + for (String fld : cleanThis) { + values.remove(fld); + } + } + } + } + } + + private String getNameInRecord(String name, MethodFrameWrapper frameWrapper, Node n) { + if (frameWrapper == null || !n.useFqn) return name; + StringBuilder sb = new StringBuilder(); + frameWrapper.prependName(sb); + return sb.append(DELIM).append(name).toString(); + } + + private boolean isRecord() { + return isRecord; + } + + + private void putValue(Map<String, Object> values, String fieldName, Object o) { + if (o == null) return; + Object val = values.get(fieldName); + if (val == null) { + values.put(fieldName, o); + return; + } + if (val instanceof List) { + List list = (List) val; + list.add(o); + return; + } + ArrayList l = new ArrayList(); + l.add(val); + l.add(o); + values.put(fieldName, l); + } + + + @Override + public String toString() { + return name; + } + } // end of class Node + + + /** + * The path is split into segments using the '/' as a seperator. However + * this method deals with special cases where there is a slash '/' character + * inside the attribute value e.g. x/@html='text/html'. We split by '/' but + * then reassemble things were the '/' appears within a quoted sub-string. + * <p> + * We have already enforced that the string must begin with a seperator. This + * method depends heavily on how split behaves if the string starts with the + * seperator or if a sequence of multiple seperator's appear. + */ + private static List<String> splitEscapeQuote(String str) { + List<String> result = new LinkedList<>(); + String[] ss = str.split("/"); + for (int i = 0; i < ss.length; i++) { // i=1: skip seperator at start of string + StringBuilder sb = new StringBuilder(); + int quoteCount = 0; + while (true) { + sb.append(ss[i]); + for (int j = 0; j < ss[i].length(); j++) + if (ss[i].charAt(j) == '\'') quoteCount++; + // have we got a split inside quoted sub-string? + if ((quoteCount % 2) == 0) break; + // yes!; replace the '/' and loop to concat next token + i++; + sb.append("/"); + } + result.add(sb.toString()); + } + return result; + } + + + /** + * Implement this interface to stream records as and when one is found. + */ + public static interface Handler { + /** + * @param record The record map. The key is the field name as provided in + * the addField() methods. The value can be a single String (for single + * valued fields) or a List<String> (for multiValued). + * @param path The forEach path for which this record is being emitted + * If there is any change all parsing will be aborted and the Exception + * is propagated up + */ + public void handle(Map<String, Object> record, String path); + } + + public static Object parseSingleFieldValue(int ev, JSONParser parser, MethodFrameWrapper runnable) throws IOException { + switch (ev) { + case STRING: + return parser.getString(); + case LONG: + return parser.getLong(); + case NUMBER: + return parser.getDouble(); + case BIGNUMBER: + return parser.getNumberChars().toString(); + case BOOLEAN: + return parser.getBoolean(); + case NULL: + parser.getNull(); + return null; + case ARRAY_START: + return parseArrayFieldValue(ev, parser, runnable); + case OBJECT_START: + if (runnable != null) { + runnable.walk(OBJECT_START); + return null; + } + consumeTillMatchingEnd(parser, 1, 0); + return null; + default: + throw new RuntimeException("Error parsing JSON field value. Unexpected " + JSONParser.getEventString(ev)); + } + } + + static abstract class MethodFrameWrapper { + Node node; + MethodFrameWrapper parent; + String name; + + void prependName(StringBuilder sb) { + if (parent != null) { + parent.prependName(sb); + sb.append(DELIM); + } + sb.append(name); + } + + public abstract void walk(int event) throws IOException; + } + + public static List<Object> parseArrayFieldValue(int ev, JSONParser parser, MethodFrameWrapper runnable) throws IOException { + assert ev == ARRAY_START; + + ArrayList lst = new ArrayList(2); + for (; ; ) { + ev = parser.nextEvent(); + if (ev == ARRAY_END) { + if (lst.isEmpty()) return null; + return lst; + } + Object val = parseSingleFieldValue(ev, parser, runnable); + if (val != null) lst.add(val); + } + } + + public static void consumeTillMatchingEnd(JSONParser parser, int obj, int arr) throws IOException { + for (; ; ) { + int event = parser.nextEvent(); + if (event == OBJECT_START) obj++; + if (event == OBJECT_END) obj--; + assert obj >= 0; + if (event == ARRAY_START) arr++; + if (event == ARRAY_END) arr--; + assert arr >= 0; + if (obj == 0 && arr == 0) break; + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/NamedList.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/NamedList.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/NamedList.java new file mode 100644 index 0000000..c539b2b --- /dev/null +++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/NamedList.java @@ -0,0 +1,708 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.common.util; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.solr.common.SolrException; + +/** + * A simple container class for modeling an ordered list of name/value pairs. + * + * <p> + * Unlike Maps: + * </p> + * <ul> + * <li>Names may be repeated</li> + * <li>Order of elements is maintained</li> + * <li>Elements may be accessed by numeric index</li> + * <li>Names and Values can both be null</li> + * </ul> + * + * <p> + * A NamedList provides fast access by element number, but not by name. + * </p> + * <p> + * When a NamedList is serialized, order is considered more important than access + * by key, so ResponseWriters that output to a format such as JSON will normally + * choose a data structure that allows order to be easily preserved in various + * clients (i.e. not a straight map). + * If access by key is more important for serialization, see {@link SimpleOrderedMap}, + * or simply use a regular {@link Map} + * </p> + * + */ +public class NamedList<T> implements Cloneable, Serializable, Iterable<Map.Entry<String,T>> { + + private static final long serialVersionUID = 1957981902839867821L; + protected final List<Object> nvPairs; + + /** Creates an empty instance */ + public NamedList() { + nvPairs = new ArrayList<>(); + } + + /** + * Creates a NamedList instance containing the "name,value" pairs contained in the + * Entry[]. + * + * <p> + * Modifying the contents of the Entry[] after calling this constructor may change + * the NamedList (in future versions of Solr), but this is not guaranteed and should + * not be relied upon. To modify the NamedList, refer to {@link #add(String, Object)} + * or {@link #remove(String)}. + * </p> + * + * @param nameValuePairs the name value pairs + */ + public NamedList(Map.Entry<String, ? extends T>[] nameValuePairs) { + nvPairs = nameValueMapToList(nameValuePairs); + } + + /** + * Creates a NamedList instance containing the "name,value" pairs contained in the + * Map. + * + * <p> + * Modifying the contents of the Map after calling this constructor may change + * the NamedList (in future versions of Solr), but this is not guaranteed and should + * not be relied upon. To modify the NamedList, refer to {@link #add(String, Object)} + * or {@link #remove(String)}. + * </p> + * + * @param nameValueMap the name value pairs + */ + public NamedList(Map<String,? extends T> nameValueMap) { + if (null == nameValueMap) { + nvPairs = new ArrayList<>(); + } else { + nvPairs = new ArrayList<>(nameValueMap.size()); + for (Map.Entry<String,? extends T> ent : nameValueMap.entrySet()) { + nvPairs.add(ent.getKey()); + nvPairs.add(ent.getValue()); + } + } + } + + /** + * Creates an instance backed by an explicitly specified list of + * pairwise names/values. + * + * <p> + * When using this constructor, runtime type safety is only guaranteed if + * all even numbered elements of the input list are of type "T". + * </p> + * + * @param nameValuePairs underlying List which should be used to implement a NamedList + * @deprecated Use {@link #NamedList(java.util.Map.Entry[])} for the NamedList instantiation + */ + @Deprecated + public NamedList(List<Object> nameValuePairs) { + nvPairs=nameValuePairs; + } + + /** + * Method to serialize Map.Entry<String, ?> to a List in which the even + * indexed elements (0,2,4. ..etc) are Strings and odd elements (1,3,5,) are of + * the type "T". + * + * @return Modified List as per the above description + * @deprecated This a temporary placeholder method until the guts of the class + * are actually replaced by List<String, ?>. + * @see <a href="https://issues.apache.org/jira/browse/SOLR-912";>SOLR-912</a> + */ + @Deprecated + private List<Object> nameValueMapToList(Map.Entry<String, ? extends T>[] nameValuePairs) { + List<Object> result = new ArrayList<>(); + for (Map.Entry<String, ?> ent : nameValuePairs) { + result.add(ent.getKey()); + result.add(ent.getValue()); + } + return result; + } + + /** The total number of name/value pairs */ + public int size() { + return nvPairs.size() >> 1; + } + + /** + * The name of the pair at the specified List index + * + * @return null if no name exists + */ + public String getName(int idx) { + return (String)nvPairs.get(idx << 1); + } + + /** + * The value of the pair at the specified List index + * + * @return may be null + */ + @SuppressWarnings("unchecked") + public T getVal(int idx) { + return (T)nvPairs.get((idx << 1) + 1); + } + + /** + * Adds a name/value pair to the end of the list. + */ + public void add(String name, T val) { + nvPairs.add(name); + nvPairs.add(val); + } + + /** + * Modifies the name of the pair at the specified index. + */ + public void setName(int idx, String name) { + nvPairs.set(idx<<1, name); + } + + /** + * Modifies the value of the pair at the specified index. + * + * @return the value that used to be at index + */ + public T setVal(int idx, T val) { + int index = (idx<<1)+1; + @SuppressWarnings("unchecked") + T old = (T)nvPairs.get( index ); + nvPairs.set(index, val); + return old; + } + + /** + * Removes the name/value pair at the specified index. + * + * @return the value at the index removed + */ + public T remove(int idx) { + int index = (idx<<1); + nvPairs.remove(index); + @SuppressWarnings("unchecked") + T result = (T)nvPairs.remove(index); // same index, as things shifted in previous remove + return result; + } + + /** + * Scans the list sequentially beginning at the specified index and + * returns the index of the first pair with the specified name. + * + * @param name name to look for, may be null + * @param start index to begin searching from + * @return The index of the first matching pair, -1 if no match + */ + public int indexOf(String name, int start) { + int sz = size(); + for (int i=start; i<sz; i++) { + String n = getName(i); + if (name==null) { + if (n==null) return i; // matched null + } else if (name.equals(n)) { + return i; + } + } + return -1; + } + + /** + * Gets the value for the first instance of the specified name + * found. + * <p> + * NOTE: this runs in linear time (it scans starting at the + * beginning of the list until it finds the first pair with + * the specified name). + * + * @return null if not found or if the value stored was null. + * @see #indexOf + * @see #get(String,int) + * + */ + public T get(String name) { + return get(name,0); + } + + /** + * Gets the value for the first instance of the specified name + * found starting at the specified index. + * <p> + * NOTE: this runs in linear time (it scans starting at the + * specified position until it finds the first pair with + * the specified name). + * + * @return null if not found or if the value stored was null. + * @see #indexOf + */ + public T get(String name, int start) { + int sz = size(); + for (int i=start; i<sz; i++) { + String n = getName(i); + if (name==null) { + if (n==null) return getVal(i); + } else if (name.equals(n)) { + return getVal(i); + } + } + return null; + } + + /** + * Gets the values for the the specified name + * + * @param name Name + * @return List of values + */ + public List<T> getAll(String name) { + List<T> result = new ArrayList<>(); + int sz = size(); + for (int i = 0; i < sz; i++) { + String n = getName(i); + if (name==n || (name!=null && name.equals(n))) { + result.add(getVal(i)); + } + } + return result; + } + + /** + * Removes all values matching the specified name + * + * @param name Name + */ + private void killAll(String name) { + int sz = size(); + // Go through the list backwards, removing matches as found. + for (int i = sz - 1; i >= 0; i--) { + String n = getName(i); + if (name==n || (name!=null && name.equals(n))) { + remove(i); + } + } + } + + /** + * Recursively parses the NamedList structure to arrive at a specific element. + * As you descend the NamedList tree, the last element can be any type, + * including NamedList, but the previous elements MUST be NamedList objects + * themselves. A null value is returned if the indicated hierarchy doesn't + * exist, but NamedList allows null values so that could be the actual value + * at the end of the path. + * + * This method is particularly useful for parsing the response from Solr's + * /admin/mbeans handler, but it also works for any complex structure. + * + * Explicitly casting the return value is recommended. An even safer option is + * to accept the return value as an object and then check its type. + * + * Usage examples: + * + * String coreName = (String) response.findRecursive + * ("solr-mbeans", "CORE", "core", "stats", "coreName"); + * long numDoc = (long) response.findRecursive + * ("solr-mbeans", "CORE", "searcher", "stats", "numDocs"); + * + * @param args + * One or more strings specifying the tree to navigate. + * @return the last entry in the given path hierarchy, null if not found. + */ + public Object findRecursive(String... args) { + NamedList<?> currentList = null; + Object value = null; + for (int i = 0; i < args.length; i++) { + String key = args[i]; + /* + * The first time through the loop, the current list is null, so we assign + * it to this list. Then we retrieve the first key from this list and + * assign it to value. + * + * On the next loop, we check whether the retrieved value is a NamedList. + * If it is, then we drop down to that NamedList, grab the value of the + * next key, and start the loop over. If it is not a NamedList, then we + * assign the value to null and break out of the loop. + * + * Assigning the value to null and then breaking out of the loop seems + * like the wrong thing to do, but there's a very simple reason that it + * works: If we have reached the last key, then the loop ends naturally + * after we retrieve the value, and that code is never executed. + */ + if (currentList == null) { + currentList = this; + } else { + if (value instanceof NamedList) { + currentList = (NamedList<?>) value; + } else { + value = null; + break; + } + } + /* + * We do not need to do a null check on currentList for the following + * assignment. The instanceof check above will fail if the current list is + * null, and if that happens, the loop will end before this point. + */ + value = currentList.get(key, 0); + } + return value; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append('{'); + int sz = size(); + for (int i=0; i<sz; i++) { + if (i != 0) sb.append(','); + sb.append(getName(i)); + sb.append('='); + sb.append(getVal(i)); + } + sb.append('}'); + + return sb.toString(); + } + + public NamedList getImmutableCopy() { + NamedList copy = clone(); + return new NamedList<>( Collections.unmodifiableList(copy.nvPairs)); + } + + public Map asMap(int maxDepth) { + LinkedHashMap result = new LinkedHashMap(); + for(int i=0;i<size();i++){ + Object val = getVal(i); + if (val instanceof NamedList && maxDepth> 0) { + //the maxDepth check is to avoid stack overflow due to infinite recursion + val = ((NamedList) val).asMap(maxDepth-1); + } + Object old = result.put(getName(i), val); + if(old!=null){ + if (old instanceof List) { + List list = (List) old; + list.add(val); + result.put(getName(i),old); + } else { + ArrayList l = new ArrayList(); + l.add(old); + l.add(val); + result.put(getName(i), l); + } + } + } + return result; + } + + /** + * + * Helper class implementing Map.Entry<String, T> to store the key-value + * relationship in NamedList (the keys of which are String-s) + */ + public static final class NamedListEntry<T> implements Map.Entry<String,T> { + + public NamedListEntry() { + + } + + public NamedListEntry(String _key, T _value) { + key = _key; + value = _value; + } + + @Override + public String getKey() { + return key; + } + + @Override + public T getValue() { + return value; + } + + @Override + public T setValue(T _value) { + T oldValue = value; + value = _value; + return oldValue; + } + + private String key; + + private T value; + } + + /** + * Iterates over the Map and sequentially adds its key/value pairs + */ + public boolean addAll(Map<String,T> args) { + for (Map.Entry<String, T> entry : args.entrySet() ) { + add(entry.getKey(), entry.getValue()); + } + return args.size()>0; + } + + /** Appends the elements of the given NamedList to this one. */ + public boolean addAll(NamedList<T> nl) { + nvPairs.addAll(nl.nvPairs); + return nl.size()>0; + } + + /** + * Makes a <i>shallow copy</i> of the named list. + */ + @Override + public NamedList<T> clone() { + ArrayList<Object> newList = new ArrayList<>(nvPairs.size()); + newList.addAll(nvPairs); + return new NamedList<>(newList); + } + + //---------------------------------------------------------------------------- + // Iterable interface + //---------------------------------------------------------------------------- + + /** + * Support the Iterable interface + */ + @Override + public Iterator<Map.Entry<String,T>> iterator() { + + final NamedList<T> list = this; + + Iterator<Map.Entry<String,T>> iter = new Iterator<Map.Entry<String,T>>() { + + int idx = 0; + + @Override + public boolean hasNext() { + return idx < list.size(); + } + + @Override + public Map.Entry<String,T> next() { + final int index = idx++; + Map.Entry<String,T> nv = new Map.Entry<String,T>() { + @Override + public String getKey() { + return list.getName( index ); + } + + @Override + public T getValue() { + return list.getVal( index ); + } + + @Override + public String toString() { + return getKey()+"="+getValue(); + } + + @Override + public T setValue(T value) { + return list.setVal(index, value); + } + }; + return nv; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + return iter; + } + + /** + * NOTE: this runs in linear time (it scans starting at the + * beginning of the list until it finds the first pair with + * the specified name). + */ + public T remove(String name) { + int idx = indexOf(name, 0); + if(idx != -1) return remove(idx); + return null; + } + + /** + * Removes and returns all values for the specified name. Returns null if + * no matches found. This method will return all matching objects, + * regardless of data type. If you are parsing Solr config options, the + * {@link #removeConfigArgs(String)} or {@link #removeBooleanArg(String)} + * methods will probably work better. + * + * @param name Name + * @return List of values + */ + public List<T> removeAll(String name) { + List<T> result = new ArrayList<>(); + result = getAll(name); + if (result.size() > 0 ) { + killAll(name); + return result; + } + return null; + } + + /** + * Used for getting a boolean argument from a NamedList object. If the name + * is not present, returns null. If there is more than one value with that + * name, or if the value found is not a Boolean or a String, throws an + * exception. If there is only one value present and it is a Boolean or a + * String, the value is removed and returned as a Boolean. If an exception + * is thrown, the NamedList is not modified. See {@link #removeAll(String)} + * and {@link #removeConfigArgs(String)} for additional ways of gathering + * configuration information from a NamedList. + * + * @param name + * The key to look up in the NamedList. + * @return The boolean value found. + * @throws SolrException + * If multiple values are found for the name or the value found is + * not a Boolean or a String. + */ + public Boolean removeBooleanArg(final String name) { + Boolean bool = getBooleanArg(name); + if (null != bool) { + remove(name); + } + return bool; + } + + /** + * Used for getting a boolean argument from a NamedList object. If the name + * is not present, returns null. If there is more than one value with that + * name, or if the value found is not a Boolean or a String, throws an + * exception. If there is only one value present and it is a Boolean or a + * String, the value is returned as a Boolean. The NamedList is not + * modified. See {@link #remove(String)}, {@link #removeAll(String)} + * and {@link #removeConfigArgs(String)} for additional ways of gathering + * configuration information from a NamedList. + * + * @param name The key to look up in the NamedList. + * @return The boolean value found. + * @throws SolrException + * If multiple values are found for the name or the value found is + * not a Boolean or a String. + */ + public Boolean getBooleanArg(final String name) { + Boolean bool; + List<T> values = getAll(name); + if (0 == values.size()) { + return null; + } + if (values.size() > 1) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Only one '" + name + "' is allowed"); + } + Object o = get(name); + if (o instanceof Boolean) { + bool = (Boolean)o; + } else if (o instanceof CharSequence) { + bool = Boolean.parseBoolean(o.toString()); + } else { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "'" + name + "' must have type Boolean or CharSequence; found " + o.getClass()); + } + return bool; + } + + /** + * Used for getting one or many arguments from NamedList objects that hold + * configuration parameters. Finds all entries in the NamedList that match + * the given name. If they are all strings or arrays of strings, remove them + * from the NamedList and return the individual elements as a {@link Collection}. + * Parameter order will be preserved if the returned collection is handled as + * an {@link ArrayList}. Throws SolrException if any of the values associated + * with the name are not strings or arrays of strings. If exception is + * thrown, the NamedList is not modified. Returns an empty collection if no + * matches found. If you need to remove and retrieve all matching items from + * the NamedList regardless of data type, use {@link #removeAll(String)} instead. + * The {@link #removeBooleanArg(String)} method can be used for retrieving a + * boolean argument. + * + * @param name + * The key to look up in the NamedList. + * @return A collection of the values found. + * @throws SolrException + * If values are found for the input key that are not strings or + * arrays of strings. + */ + @SuppressWarnings("rawtypes") + public Collection<String> removeConfigArgs(final String name) + throws SolrException { + List<T> objects = getAll(name); + List<String> collection = new ArrayList<>(size() / 2); + final String err = "init arg '" + name + "' must be a string " + + "(ie: 'str'), or an array (ie: 'arr') containing strings; found: "; + + for (Object o : objects) { + if (o instanceof String) { + collection.add((String) o); + continue; + } + + // If it's an array, convert to List (which is a Collection). + if (o instanceof Object[]) { + o = Arrays.asList((Object[]) o); + } + + // If it's a Collection, collect each value. + if (o instanceof Collection) { + for (Object item : (Collection) o) { + if (!(item instanceof String)) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, err + item.getClass()); + } + collection.add((String) item); + } + continue; + } + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, err + o.getClass()); + } + + if (collection.size() > 0) { + killAll(name); + } + + return collection; + } + + public void clear() { + nvPairs.clear(); + } + + @Override + public int hashCode() { + return nvPairs.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof NamedList)) return false; + NamedList<?> nl = (NamedList<?>) obj; + return this.nvPairs.equals(nl.nvPairs); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/ObjectReleaseTracker.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/ObjectReleaseTracker.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/ObjectReleaseTracker.java new file mode 100644 index 0000000..47ab21a --- /dev/null +++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/ObjectReleaseTracker.java @@ -0,0 +1,62 @@ +package org.apache.solr.common.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +public class ObjectReleaseTracker { + public static Map<Object,String> OBJECTS = new ConcurrentHashMap<>(); + + public static boolean track(Object object) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + new ObjectTrackerException().printStackTrace(pw); + OBJECTS.put(object, sw.toString()); + return true; + } + + public static boolean release(Object object) { + OBJECTS.remove(object); + return true; + } + + public static boolean clearObjectTrackerAndCheckEmpty() { + Set<Entry<Object,String>> entries = OBJECTS.entrySet(); + boolean empty = entries.isEmpty(); + if (entries.size() > 0) { + System.err.println("ObjectTracker found objects that were not released!!!"); + } + + for (Entry<Object,String> entry : entries) { + System.err.println(entry.getValue()); + } + + OBJECTS.clear(); + + return empty; + } + + private static class ObjectTrackerException extends RuntimeException { + + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/RetryUtil.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/RetryUtil.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/RetryUtil.java new file mode 100644 index 0000000..83ee100 --- /dev/null +++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/RetryUtil.java @@ -0,0 +1,43 @@ +package org.apache.solr.common.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.concurrent.TimeUnit; + +public class RetryUtil { + public static interface RetryCmd { + public void execute() throws Throwable; + } + + public static void retryOnThrowable(Class clazz, long timeoutms, long intervalms, RetryCmd cmd) throws Throwable { + long timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeoutms, TimeUnit.MILLISECONDS); + while (true) { + try { + cmd.execute(); + } catch (Throwable t) { + if (clazz.isInstance(t) && System.nanoTime() < timeout) { + Thread.sleep(intervalms); + continue; + } + throw t; + } + // success + break; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/SimpleOrderedMap.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/SimpleOrderedMap.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/SimpleOrderedMap.java new file mode 100644 index 0000000..c9996d1 --- /dev/null +++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/SimpleOrderedMap.java @@ -0,0 +1,67 @@ +package org.apache.solr.common.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.*; + + +/** <code>SimpleOrderedMap</code> is a {@link NamedList} where access by key is more + * important than maintaining order when it comes to representing the + * held data in other forms, as ResponseWriters normally do. + * It's normally not a good idea to repeat keys or use null keys, but this + * is not enforced. If key uniqueness enforcement is desired, use a regular {@link Map}. + * <p> + * For example, a JSON response writer may choose to write a SimpleOrderedMap + * as {"foo":10,"bar":20} and may choose to write a NamedList as + * ["foo",10,"bar",20]. An XML response writer may choose to render both + * the same way. + * </p> + * <p> + * This class does not provide efficient lookup by key, its main purpose is + * to hold data to be serialized. It aims to minimize overhead and to be + * efficient at adding new elements. + * </p> + */ +public class SimpleOrderedMap<T> extends NamedList<T> { + /** Creates an empty instance */ + public SimpleOrderedMap() { + super(); + } + + /** + * Creates an instance backed by an explicitly specified list of + * pairwise names/values. + * + * @param nameValuePairs underlying List which should be used to implement a SimpleOrderedMap; modifying this List will affect the SimpleOrderedMap. + */ + @Deprecated + public SimpleOrderedMap(List<Object> nameValuePairs) { + super(nameValuePairs); + } + + public SimpleOrderedMap(Map.Entry<String, T>[] nameValuePairs) { + super(nameValuePairs); + } + + @Override + public SimpleOrderedMap<T> clone() { + ArrayList<Object> newList = new ArrayList<>(nvPairs.size()); + newList.addAll(nvPairs); + return new SimpleOrderedMap<>(newList); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/SolrjNamedThreadFactory.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/SolrjNamedThreadFactory.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/SolrjNamedThreadFactory.java new file mode 100644 index 0000000..2a7c901 --- /dev/null +++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/SolrjNamedThreadFactory.java @@ -0,0 +1,50 @@ +package org.apache.solr.common.util; + +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class SolrjNamedThreadFactory implements ThreadFactory { + private static final AtomicInteger poolNumber = new AtomicInteger(1); + private final ThreadGroup group; + private final AtomicInteger threadNumber = new AtomicInteger(1); + private final String prefix; + + public SolrjNamedThreadFactory(String namePrefix) { + SecurityManager s = System.getSecurityManager(); + group = (s != null)? s.getThreadGroup() : + Thread.currentThread().getThreadGroup(); + prefix = namePrefix + "-" + + poolNumber.getAndIncrement() + + "-thread-"; + } + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(group, r, + prefix + threadNumber.getAndIncrement(), + 0); + + t.setDaemon(false); + + if (t.getPriority() != Thread.NORM_PRIORITY) + t.setPriority(Thread.NORM_PRIORITY); + return t; + } +} \ No newline at end of file
