sumitagrawl commented on code in PR #7213: URL: https://github.com/apache/ozone/pull/7213#discussion_r1812967875
########## hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java: ########## @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.ozone.recon; + +import com.google.inject.Inject; +import org.jooq.DSLContext; +import org.jooq.impl.DSL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.sql.DataSource; +import java.sql.SQLException; + +import static org.jooq.impl.DSL.name; + +/** + * Manager for handling the Recon Schema Version table. + * This class provides methods to get and update the current schema version. + */ +public class ReconSchemaVersionTableManager { + + private static final Logger LOG = LoggerFactory.getLogger(ReconSchemaVersionTableManager.class); + public static final String RECON_SCHEMA_VERSION_TABLE_NAME = "RECON_SCHEMA_VERSION"; + private final DSLContext dslContext; + private final DataSource dataSource; + + @Inject + public ReconSchemaVersionTableManager(DataSource dataSource) throws SQLException { + this.dataSource = dataSource; + this.dslContext = DSL.using(dataSource.getConnection()); + } + + /** + * Get the current schema version from the RECON_SCHEMA_VERSION table. + * If the table is empty, or if it does not exist, it will return 0. + * @return The current schema version. + */ + public int getCurrentSchemaVersion() { + try { + return dslContext.select(DSL.field(name("version_number"))) + .from(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) + .fetchOptional() + .map(record -> record.get( + DSL.field(name("version_number"), Integer.class))) + .orElse(-1); // Return -1 if no version is found + } catch (Exception e) { + LOG.error("Failed to fetch the current schema version.", e); + return 0; // Return 0 if there is an exception Review Comment: we may need throw exception if unable to read table. "0" may give wrong data. ########## hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java: ########## @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.ozone.recon.upgrade; + +import org.apache.hadoop.ozone.recon.ReconSchemaVersionTableManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * ReconLayoutVersionManager is responsible for managing the layout version of the Recon service. + * It determines the current Metadata Layout Version (MLV) and Software Layout Version (SLV) of the + * Recon service, and finalizes the layout features that need to be upgraded. + */ +public class ReconLayoutVersionManager { + + private static final Logger LOG = LoggerFactory.getLogger(ReconLayoutVersionManager.class); + + private final ReconSchemaVersionTableManager schemaVersionTableManager; + + // Metadata Layout Version (MLV) of the Recon Metadata on disk + private int currentMLV; + // Software Layout Version (SLV) of the Recon service + private final int currentSLV; + + public ReconLayoutVersionManager(ReconSchemaVersionTableManager schemaVersionTableManager) { + this.schemaVersionTableManager = schemaVersionTableManager; + this.currentMLV = determineMLV(); + this.currentSLV = determineSLV(); + ReconLayoutFeature.registerUpgradeActions(); // Register actions via annotation + } + + /** + * Determines the current Metadata Layout Version (MLV) from the version table. + * @return The current Metadata Layout Version (MLV). + */ + private int determineMLV() { + return schemaVersionTableManager.getCurrentSchemaVersion(); + } + + /** + * Determines the Software Layout Version (SLV) based on the latest feature version. + * @return The Software Layout Version (SLV). + */ + private int determineSLV() { + return Arrays.stream(ReconLayoutFeature.values()) + .mapToInt(ReconLayoutFeature::getVersion) + .max() + .orElse(0); // Default to 0 if no features are defined + } + + /** + * Finalizes the layout features that need to be upgraded, by executing the upgrade action for each + * feature that is registered for finalization. + */ + public void finalizeLayoutFeatures() { + // Get features that need finalization, sorted by version + List<ReconLayoutFeature> featuresToFinalize = getRegisteredFeatures(); + + for (ReconLayoutFeature feature : featuresToFinalize) { + try { + // If the feature is INITIAL_VERSION, skip executing any action and just update the schema version + if (feature == ReconLayoutFeature.INITIAL_VERSION) { + updateSchemaVersion(0); Review Comment: It should call action if any are there registered. If not registered, it will not come in loop. Do not need any special handling. ########## hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutVersionManager.java: ########## @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.ozone.recon.upgrade; + +import org.apache.hadoop.ozone.recon.ReconSchemaVersionTableManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * ReconLayoutVersionManager is responsible for managing the layout version of the Recon service. + * It determines the current Metadata Layout Version (MLV) and Software Layout Version (SLV) of the + * Recon service, and finalizes the layout features that need to be upgraded. + */ +public class ReconLayoutVersionManager { + + private static final Logger LOG = LoggerFactory.getLogger(ReconLayoutVersionManager.class); + + private final ReconSchemaVersionTableManager schemaVersionTableManager; + + // Metadata Layout Version (MLV) of the Recon Metadata on disk + private int currentMLV; + // Software Layout Version (SLV) of the Recon service + private final int currentSLV; + + public ReconLayoutVersionManager(ReconSchemaVersionTableManager schemaVersionTableManager) { + this.schemaVersionTableManager = schemaVersionTableManager; + this.currentMLV = determineMLV(); + this.currentSLV = determineSLV(); + ReconLayoutFeature.registerUpgradeActions(); // Register actions via annotation + } + + /** + * Determines the current Metadata Layout Version (MLV) from the version table. + * @return The current Metadata Layout Version (MLV). + */ + private int determineMLV() { + return schemaVersionTableManager.getCurrentSchemaVersion(); + } + + /** + * Determines the Software Layout Version (SLV) based on the latest feature version. + * @return The Software Layout Version (SLV). + */ + private int determineSLV() { + return Arrays.stream(ReconLayoutFeature.values()) + .mapToInt(ReconLayoutFeature::getVersion) + .max() + .orElse(0); // Default to 0 if no features are defined + } + + /** + * Finalizes the layout features that need to be upgraded, by executing the upgrade action for each + * feature that is registered for finalization. + */ + public void finalizeLayoutFeatures() { + // Get features that need finalization, sorted by version + List<ReconLayoutFeature> featuresToFinalize = getRegisteredFeatures(); + + for (ReconLayoutFeature feature : featuresToFinalize) { + try { + // If the feature is INITIAL_VERSION, skip executing any action and just update the schema version + if (feature == ReconLayoutFeature.INITIAL_VERSION) { + updateSchemaVersion(0); + LOG.info("INITIAL_VERSION feature processed by setting schema version to 0."); + continue; + } + + // Fetch only the FINALIZE action for the feature + Optional<ReconUpgradeAction> action = feature.getAction(ReconUpgradeAction.UpgradeActionType.FINALIZE); + if (action.isPresent()) { + // Execute the upgrade action & update the schema version in the DB + action.get().execute(); + updateSchemaVersion(feature.getVersion()); + LOG.info("Feature versioned {} finalized successfully.", feature.getVersion()); + } + } catch (Exception e) { + LOG.error("Failed to finalize feature {}: {}", feature.getVersion(), e.getMessage()); + break; + } + } + } + + /** + * Returns a list of ReconLayoutFeature objects that are registered for finalization. + */ + protected List<ReconLayoutFeature> getRegisteredFeatures() { + List<ReconLayoutFeature> allFeatures = + Arrays.asList(ReconLayoutFeature.values()); + + LOG.info("Current MLV: {}. SLV: {}. Checking features for registration...", currentMLV, currentSLV); + + List<ReconLayoutFeature> registeredFeatures = allFeatures.stream() + .filter(feature -> feature.getVersion() > currentMLV) + .sorted((a, b) -> Integer.compare(a.getVersion(), b.getVersion())) // Sort by version in ascending order + .collect(Collectors.toList()); + + return registeredFeatures; + } + + /** + * Updates the Software Layout Version (SLV) in the database after finalizing a feature. + * @param newVersion The new Software Layout Version (SLV) to set. + */ + private void updateSchemaVersion(int newVersion) { + // Logic to update the MLV in the database + this.currentMLV = newVersion; Review Comment: setting currentMLV can be after db update. if DB have some exception, this flag is not required to be updated. ########## hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java: ########## @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.ozone.recon; + +import com.google.inject.Inject; +import org.jooq.DSLContext; +import org.jooq.impl.DSL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.sql.DataSource; +import java.sql.SQLException; + +import static org.jooq.impl.DSL.name; + +/** + * Manager for handling the Recon Schema Version table. + * This class provides methods to get and update the current schema version. + */ +public class ReconSchemaVersionTableManager { + + private static final Logger LOG = LoggerFactory.getLogger(ReconSchemaVersionTableManager.class); + public static final String RECON_SCHEMA_VERSION_TABLE_NAME = "RECON_SCHEMA_VERSION"; + private final DSLContext dslContext; + private final DataSource dataSource; + + @Inject + public ReconSchemaVersionTableManager(DataSource dataSource) throws SQLException { + this.dataSource = dataSource; + this.dslContext = DSL.using(dataSource.getConnection()); + } + + /** + * Get the current schema version from the RECON_SCHEMA_VERSION table. + * If the table is empty, or if it does not exist, it will return 0. + * @return The current schema version. + */ + public int getCurrentSchemaVersion() { + try { + return dslContext.select(DSL.field(name("version_number"))) + .from(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) + .fetchOptional() + .map(record -> record.get( + DSL.field(name("version_number"), Integer.class))) + .orElse(-1); // Return -1 if no version is found + } catch (Exception e) { + LOG.error("Failed to fetch the current schema version.", e); + return 0; // Return 0 if there is an exception + } + } + + /** + * Update the schema version in the RECON_SCHEMA_VERSION table after all tables are upgraded. + * + * @param newVersion The new version to set. + */ + public void updateSchemaVersion(int newVersion) { + try { + boolean recordExists = dslContext.fetchExists(dslContext.selectOne() + .from(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME))); + + if (recordExists) { + // Update the existing schema version record + dslContext.update(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) + .set(DSL.field(name("version_number")), newVersion) + .set(DSL.field(name("applied_on")), DSL.currentTimestamp()) + .execute(); + LOG.info("Updated schema version to '{}'.", newVersion); + } else { + // Insert a new schema version record + dslContext.insertInto(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)) + .columns(DSL.field(name("version_number")), + DSL.field(name("applied_on"))) + .values(newVersion, DSL.currentTimestamp()) + .execute(); + LOG.info("Inserted new schema version '{}'.", newVersion); + } + } catch (Exception e) { + LOG.error("Failed to update schema version to '{}'.", newVersion, e); Review Comment: Need throw exception if unable to update. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
