Author: tommaso
Date: Wed Dec 19 09:26:50 2018
New Revision: 1849279
URL: http://svn.apache.org/viewvc?rev=1849279&view=rev
Log:
OAK-7924 - added compressing codec
Added:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/CompressingCodec.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakCodecTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/CompressingCodecTest.java
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java
Added:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/CompressingCodec.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/CompressingCodec.java?rev=1849279&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/CompressingCodec.java
(added)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/CompressingCodec.java
Wed Dec 19 09:26:50 2018
@@ -0,0 +1,93 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+package org.apache.jackrabbit.oak.plugins.index.lucene.util;
+
+import org.apache.lucene.codecs.*;
+import org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat;
+import org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat;
+import org.apache.lucene.codecs.compressing.CompressionMode;
+import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
+import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
+import org.apache.lucene.codecs.lucene46.Lucene46Codec;
+import org.apache.lucene.codecs.lucene46.Lucene46FieldInfosFormat;
+import org.apache.lucene.codecs.lucene46.Lucene46SegmentInfoFormat;
+
+/**
+ * Lucene Codec aimed to reduce index size as much as possible by enabling
highest possible compression on term vectors and stored fields.
+ */
+public class CompressingCodec extends FilterCodec {
+
+ private static final int CHUNK_SIZE = 1024;
+ private static final String SEGMENT_SUFFIX = "ctv";
+
+ private final TermVectorsFormat vectorsFormat = new
CompressingTermVectorsFormat("Lucene41",
+ SEGMENT_SUFFIX, CompressionMode.HIGH_COMPRESSION, CHUNK_SIZE);
+ private final FieldInfosFormat fieldInfosFormat = new
Lucene46FieldInfosFormat();
+ private final SegmentInfoFormat segmentInfosFormat = new
Lucene46SegmentInfoFormat();
+ private final LiveDocsFormat liveDocsFormat = new Lucene40LiveDocsFormat();
+ private final PostingsFormat defaultFormat =
PostingsFormat.forName("Lucene41");
+ private final DocValuesFormat defaultDVFormat =
DocValuesFormat.forName("Lucene45");
+ private final NormsFormat normsFormat = new Lucene42NormsFormat();
+ private final StoredFieldsFormat fieldsFormat = new
CompressingStoredFieldsFormat("Lucene41",
+ CompressionMode.HIGH_COMPRESSION, CHUNK_SIZE);
+
+ public CompressingCodec() {
+ super("compressingCodec", new Lucene46Codec());
+ }
+
+ @Override
+ public PostingsFormat postingsFormat() {
+ return defaultFormat;
+ }
+
+ @Override
+ public DocValuesFormat docValuesFormat() {
+ return defaultDVFormat;
+ }
+
+ @Override
+ public StoredFieldsFormat storedFieldsFormat() {
+ return fieldsFormat;
+ }
+
+ @Override
+ public TermVectorsFormat termVectorsFormat() {
+ return vectorsFormat;
+ }
+
+ @Override
+ public FieldInfosFormat fieldInfosFormat() {
+ return fieldInfosFormat;
+ }
+
+ @Override
+ public SegmentInfoFormat segmentInfoFormat() {
+ return segmentInfosFormat;
+ }
+
+ @Override
+ public NormsFormat normsFormat() {
+ return normsFormat;
+ }
+
+ @Override
+ public LiveDocsFormat liveDocsFormat() {
+ return liveDocsFormat;
+ }
+}
\ No newline at end of file
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec?rev=1849279&r1=1849278&r2=1849279&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec
Wed Dec 19 09:26:50 2018
@@ -17,4 +17,5 @@ org.apache.lucene.codecs.lucene41.Lucene
org.apache.lucene.codecs.lucene42.Lucene42Codec
org.apache.lucene.codecs.lucene45.Lucene45Codec
org.apache.lucene.codecs.lucene46.Lucene46Codec
-org.apache.jackrabbit.oak.plugins.index.lucene.OakCodec
\ No newline at end of file
+org.apache.jackrabbit.oak.plugins.index.lucene.OakCodec
+org.apache.jackrabbit.oak.plugins.index.lucene.util.CompressingCodec
\ No newline at end of file
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java?rev=1849279&r1=1849278&r2=1849279&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java
Wed Dec 19 09:26:50 2018
@@ -42,11 +42,6 @@ import org.apache.jackrabbit.oak.commons
import org.apache.jackrabbit.oak.plugins.blob.BlobStoreStats;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore;
-import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
-import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexDefinition;
-import
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
-import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
-import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil;
import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.CopyOnReadDirectory;
import
org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider;
@@ -141,6 +136,12 @@ public class LuceneWritesOnSegmentStatsT
{false, "Lucene46", false, 100, "tiered"},
{false, "Lucene46", false, 100, "mitigated"},
{false, "Lucene46", false, 100, "no"},
+ {false, "compressingCodec", false, 4096, "tiered"},
+ {false, "compressingCodec", false, 4096, "mitigated"},
+ {false, "compressingCodec", false, 4096, "no"},
+ {false, "compressingCodec", false, 100, "tiered"},
+ {false, "compressingCodec", false, 100, "mitigated"},
+ {false, "compressingCodec", false, 100, "no"},
});
}
Added:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakCodecTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakCodecTest.java?rev=1849279&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakCodecTest.java
(added)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakCodecTest.java
Wed Dec 19 09:26:50 2018
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Tests for {@link OakCodec}
+ */
+public class OakCodecTest {
+
+ @Test
+ public void tesFormats() {
+ OakCodec oakCodec = new OakCodec();
+ assertNotNull(oakCodec.docValuesFormat());
+ assertNotNull(oakCodec.fieldInfosFormat());
+ assertNotNull(oakCodec.liveDocsFormat());
+ assertNotNull(oakCodec.normsFormat());
+ assertNotNull(oakCodec.postingsFormat());
+ assertNotNull(oakCodec.segmentInfoFormat());
+ assertNotNull(oakCodec.storedFieldsFormat());
+ assertNotNull(oakCodec.termVectorsFormat());
+ }
+
+}
\ No newline at end of file
Added:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/CompressingCodecTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/CompressingCodecTest.java?rev=1849279&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/CompressingCodecTest.java
(added)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/CompressingCodecTest.java
Wed Dec 19 09:26:50 2018
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene.util;
+
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.CompressingCodec;
+import org.junit.Test;
+
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Tests for {@link CompressingCodec}
+ */
+public class CompressingCodecTest {
+
+ @Test
+ public void tesFormats() {
+ CompressingCodec compressingCodec = new CompressingCodec();
+ assertNotNull(compressingCodec.docValuesFormat());
+ assertNotNull(compressingCodec.fieldInfosFormat());
+ assertNotNull(compressingCodec.liveDocsFormat());
+ assertNotNull(compressingCodec.normsFormat());
+ assertNotNull(compressingCodec.postingsFormat());
+ assertNotNull(compressingCodec.segmentInfoFormat());
+ assertNotNull(compressingCodec.storedFieldsFormat());
+ assertNotNull(compressingCodec.termVectorsFormat());
+ }
+
+}
\ No newline at end of file
