sashapolo commented on a change in pull request #468: URL: https://github.com/apache/ignite-3/pull/468#discussion_r766743157
########## File path: modules/storage-rocksdb/src/main/java/org/apache/ignite/internal/storage/rocksdb/index/RocksDbSortedIndexStorage.java ########## @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.storage.rocksdb.index; + +import org.apache.ignite.internal.rocksdb.ColumnFamily; +import org.apache.ignite.internal.rocksdb.RocksIteratorAdapter; +import org.apache.ignite.internal.schema.ByteBufferRow; +import org.apache.ignite.internal.storage.StorageException; +import org.apache.ignite.internal.storage.index.IndexRow; +import org.apache.ignite.internal.storage.index.IndexRowDeserializer; +import org.apache.ignite.internal.storage.index.IndexRowFactory; +import org.apache.ignite.internal.storage.index.IndexRowPrefix; +import org.apache.ignite.internal.storage.index.SortedIndexDescriptor; +import org.apache.ignite.internal.storage.index.SortedIndexStorage; +import org.apache.ignite.internal.util.Cursor; +import org.jetbrains.annotations.Nullable; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; + +/** + * {@link SortedIndexStorage} implementation based on RocksDB. + */ +public class RocksDbSortedIndexStorage implements SortedIndexStorage { + private final ColumnFamily indexCf; + + private final SortedIndexDescriptor descriptor; + + private final IndexRowFactory indexRowFactory; + + private final IndexRowDeserializer indexRowDeserializer; + + /** + * Creates a new Index storage. + * + * @param indexCf Column Family for storing the data. + * @param descriptor Index descriptor. + */ + public RocksDbSortedIndexStorage(ColumnFamily indexCf, SortedIndexDescriptor descriptor) { + this.indexCf = indexCf; + this.descriptor = descriptor; + this.indexRowFactory = new BinaryIndexRowFactory(descriptor); + this.indexRowDeserializer = new BinaryIndexRowDeserializer(descriptor); + } + + @Override + public SortedIndexDescriptor indexDescriptor() { + return descriptor; + } + + @Override + public IndexRowFactory indexKeyFactory() { + return indexRowFactory; + } + + @Override + public IndexRowDeserializer indexRowDeserializer() { + return indexRowDeserializer; + } + + @Override + public void put(IndexRow row) { + try { + indexCf.put(row.rowBytes(), row.primaryKey().keyBytes()); + } catch (RocksDBException e) { + throw new StorageException("Error while adding data to Rocks DB", e); + } + } + + @Override + public void remove(IndexRow key) { + try { + indexCf.delete(key.rowBytes()); + } catch (RocksDBException e) { + throw new StorageException("Error while removing data from Rocks DB", e); + } + } + + @Override + public Cursor<IndexRow> range(IndexRowPrefix lowerBound, IndexRowPrefix upperBound) { + RocksIterator iter = indexCf.newIterator(); + + iter.seekToFirst(); Review comment: Unfortunately, no. `BinaryRow` serialization places columns differently than specified in the schema. This means that some column, that is missing in the prefix, can actually be the first bytes of the serialized row. What I'm trying to say is that some prefix column values might not serialize as a byte prefix of the serialized row =( ########## File path: modules/storage-rocksdb/src/main/java/org/apache/ignite/internal/storage/rocksdb/index/RocksDbSortedIndexStorage.java ########## @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.storage.rocksdb.index; + +import org.apache.ignite.internal.rocksdb.ColumnFamily; +import org.apache.ignite.internal.rocksdb.RocksIteratorAdapter; +import org.apache.ignite.internal.schema.ByteBufferRow; +import org.apache.ignite.internal.storage.StorageException; +import org.apache.ignite.internal.storage.index.IndexRow; +import org.apache.ignite.internal.storage.index.IndexRowDeserializer; +import org.apache.ignite.internal.storage.index.IndexRowFactory; +import org.apache.ignite.internal.storage.index.IndexRowPrefix; +import org.apache.ignite.internal.storage.index.SortedIndexDescriptor; +import org.apache.ignite.internal.storage.index.SortedIndexStorage; +import org.apache.ignite.internal.util.Cursor; +import org.jetbrains.annotations.Nullable; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; + +/** + * {@link SortedIndexStorage} implementation based on RocksDB. + */ +public class RocksDbSortedIndexStorage implements SortedIndexStorage { + private final ColumnFamily indexCf; + + private final SortedIndexDescriptor descriptor; + + private final IndexRowFactory indexRowFactory; + + private final IndexRowDeserializer indexRowDeserializer; + + /** + * Creates a new Index storage. + * + * @param indexCf Column Family for storing the data. + * @param descriptor Index descriptor. + */ + public RocksDbSortedIndexStorage(ColumnFamily indexCf, SortedIndexDescriptor descriptor) { + this.indexCf = indexCf; + this.descriptor = descriptor; + this.indexRowFactory = new BinaryIndexRowFactory(descriptor); + this.indexRowDeserializer = new BinaryIndexRowDeserializer(descriptor); + } + + @Override + public SortedIndexDescriptor indexDescriptor() { + return descriptor; + } + + @Override + public IndexRowFactory indexKeyFactory() { + return indexRowFactory; + } + + @Override + public IndexRowDeserializer indexRowDeserializer() { + return indexRowDeserializer; + } + + @Override + public void put(IndexRow row) { + try { + indexCf.put(row.rowBytes(), row.primaryKey().keyBytes()); + } catch (RocksDBException e) { + throw new StorageException("Error while adding data to Rocks DB", e); + } + } + + @Override + public void remove(IndexRow key) { + try { + indexCf.delete(key.rowBytes()); + } catch (RocksDBException e) { + throw new StorageException("Error while removing data from Rocks DB", e); + } + } + + @Override + public Cursor<IndexRow> range(IndexRowPrefix lowerBound, IndexRowPrefix upperBound) { + RocksIterator iter = indexCf.newIterator(); + + iter.seekToFirst(); Review comment: I thought that I couldn't, but actually I can, let me try ########## File path: modules/storage-rocksdb/src/main/java/org/apache/ignite/internal/storage/rocksdb/index/RocksDbSortedIndexStorage.java ########## @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.storage.rocksdb.index; + +import org.apache.ignite.internal.rocksdb.ColumnFamily; +import org.apache.ignite.internal.rocksdb.RocksIteratorAdapter; +import org.apache.ignite.internal.schema.ByteBufferRow; +import org.apache.ignite.internal.storage.StorageException; +import org.apache.ignite.internal.storage.index.IndexRow; +import org.apache.ignite.internal.storage.index.IndexRowDeserializer; +import org.apache.ignite.internal.storage.index.IndexRowFactory; +import org.apache.ignite.internal.storage.index.IndexRowPrefix; +import org.apache.ignite.internal.storage.index.SortedIndexDescriptor; +import org.apache.ignite.internal.storage.index.SortedIndexStorage; +import org.apache.ignite.internal.util.Cursor; +import org.jetbrains.annotations.Nullable; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; + +/** + * {@link SortedIndexStorage} implementation based on RocksDB. + */ +public class RocksDbSortedIndexStorage implements SortedIndexStorage { + private final ColumnFamily indexCf; + + private final SortedIndexDescriptor descriptor; + + private final IndexRowFactory indexRowFactory; + + private final IndexRowDeserializer indexRowDeserializer; + + /** + * Creates a new Index storage. + * + * @param indexCf Column Family for storing the data. + * @param descriptor Index descriptor. + */ + public RocksDbSortedIndexStorage(ColumnFamily indexCf, SortedIndexDescriptor descriptor) { + this.indexCf = indexCf; + this.descriptor = descriptor; + this.indexRowFactory = new BinaryIndexRowFactory(descriptor); + this.indexRowDeserializer = new BinaryIndexRowDeserializer(descriptor); + } + + @Override + public SortedIndexDescriptor indexDescriptor() { + return descriptor; + } + + @Override + public IndexRowFactory indexKeyFactory() { + return indexRowFactory; + } + + @Override + public IndexRowDeserializer indexRowDeserializer() { + return indexRowDeserializer; + } + + @Override + public void put(IndexRow row) { + try { + indexCf.put(row.rowBytes(), row.primaryKey().keyBytes()); + } catch (RocksDBException e) { + throw new StorageException("Error while adding data to Rocks DB", e); + } + } + + @Override + public void remove(IndexRow key) { + try { + indexCf.delete(key.rowBytes()); + } catch (RocksDBException e) { + throw new StorageException("Error while removing data from Rocks DB", e); + } + } + + @Override + public Cursor<IndexRow> range(IndexRowPrefix lowerBound, IndexRowPrefix upperBound) { + RocksIterator iter = indexCf.newIterator(); + + iter.seekToFirst(); Review comment: Unfortunately no. The problem is that it will be impossible to reuse the `BinaryRowComparator`, because of `BinaryRow` serialization works: a serialized prefix and a serialized full row can have a different internal column order, therefore it will be impossible to de-serialize the prefix using the full row schema ########## File path: modules/storage-rocksdb/src/main/java/org/apache/ignite/internal/storage/rocksdb/index/RocksDbSortedIndexStorage.java ########## @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.storage.rocksdb.index; + +import org.apache.ignite.internal.rocksdb.ColumnFamily; +import org.apache.ignite.internal.rocksdb.RocksIteratorAdapter; +import org.apache.ignite.internal.schema.ByteBufferRow; +import org.apache.ignite.internal.storage.StorageException; +import org.apache.ignite.internal.storage.index.IndexRow; +import org.apache.ignite.internal.storage.index.IndexRowDeserializer; +import org.apache.ignite.internal.storage.index.IndexRowFactory; +import org.apache.ignite.internal.storage.index.IndexRowPrefix; +import org.apache.ignite.internal.storage.index.SortedIndexDescriptor; +import org.apache.ignite.internal.storage.index.SortedIndexStorage; +import org.apache.ignite.internal.util.Cursor; +import org.jetbrains.annotations.Nullable; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; + +/** + * {@link SortedIndexStorage} implementation based on RocksDB. + */ +public class RocksDbSortedIndexStorage implements SortedIndexStorage { + private final ColumnFamily indexCf; + + private final SortedIndexDescriptor descriptor; + + private final IndexRowFactory indexRowFactory; + + private final IndexRowDeserializer indexRowDeserializer; + + /** + * Creates a new Index storage. + * + * @param indexCf Column Family for storing the data. + * @param descriptor Index descriptor. + */ + public RocksDbSortedIndexStorage(ColumnFamily indexCf, SortedIndexDescriptor descriptor) { + this.indexCf = indexCf; + this.descriptor = descriptor; + this.indexRowFactory = new BinaryIndexRowFactory(descriptor); + this.indexRowDeserializer = new BinaryIndexRowDeserializer(descriptor); + } + + @Override + public SortedIndexDescriptor indexDescriptor() { + return descriptor; + } + + @Override + public IndexRowFactory indexKeyFactory() { + return indexRowFactory; + } + + @Override + public IndexRowDeserializer indexRowDeserializer() { + return indexRowDeserializer; + } + + @Override + public void put(IndexRow row) { + try { + indexCf.put(row.rowBytes(), row.primaryKey().keyBytes()); + } catch (RocksDBException e) { + throw new StorageException("Error while adding data to Rocks DB", e); + } + } + + @Override + public void remove(IndexRow key) { + try { + indexCf.delete(key.rowBytes()); + } catch (RocksDBException e) { + throw new StorageException("Error while removing data from Rocks DB", e); + } + } + + @Override + public Cursor<IndexRow> range(IndexRowPrefix lowerBound, IndexRowPrefix upperBound) { + RocksIterator iter = indexCf.newIterator(); + + iter.seekToFirst(); Review comment: Unfortunately no. The problem is that it will be impossible to reuse the `BinaryRowComparator`, because of how `BinaryRow` serialization works: a serialized prefix and a serialized full row can have a different internal column order, therefore it will be impossible to de-serialize the prefix using the full row schema ########## File path: modules/schema/src/main/java/org/apache/ignite/internal/schema/ByteBufferRow.java ########## @@ -137,7 +137,11 @@ public double readDouble(int off) { /** {@inheritDoc} */ @Override public String readString(int off, int len) { - return new String(buf.array(), off, len, StandardCharsets.UTF_8); + if (buf.hasArray()) { + return new String(buf.array(), off, len, StandardCharsets.UTF_8); Review comment: no, it won't ########## File path: modules/storage-api/src/main/java/org/apache/ignite/internal/storage/index/SortedIndexDescriptor.java ########## @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.storage.index; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.ignite.configuration.NamedListView; +import org.apache.ignite.configuration.schemas.table.ColumnView; +import org.apache.ignite.configuration.schemas.table.IndexColumnView; +import org.apache.ignite.configuration.schemas.table.SortedIndexView; +import org.apache.ignite.configuration.schemas.table.TableView; +import org.apache.ignite.internal.schema.Column; +import org.apache.ignite.internal.schema.SchemaDescriptor; +import org.apache.ignite.internal.schema.configuration.SchemaConfigurationConverter; +import org.apache.ignite.internal.schema.configuration.SchemaDescriptorConverter; +import org.apache.ignite.internal.storage.StorageException; +import org.apache.ignite.schema.definition.ColumnDefinition; + +/** + * Descriptor for creating a Sorted Index Storage. + * + * @see SortedIndexStorage + */ +public class SortedIndexDescriptor { + /** + * Descriptor of a Sorted Index column (column name and column sort order). + */ + public static class ColumnDescriptor { + private final Column column; + + private final boolean asc; + + ColumnDescriptor(Column column, boolean asc) { + this.column = column; + this.asc = asc; + } + + public Column column() { + return column; + } + + public boolean asc() { + return asc; + } + } + + private final String name; + + private final List<ColumnDescriptor> columns; + + private final SchemaDescriptor schemaDescriptor; Review comment: https://issues.apache.org/jira/browse/IGNITE-16105 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
