sv2000 commented on a change in pull request #3296: URL: https://github.com/apache/gobblin/pull/3296#discussion_r660003516
########## File path: gobblin-runtime/src/main/java/org/apache/gobblin/runtime/troubleshooter/AutomaticTroubleshooter.java ########## @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.runtime.troubleshooter; + +import java.util.List; +import java.util.Objects; + +import org.apache.commons.text.TextStringBuilder; +import org.apache.log4j.Level; +import org.apache.log4j.LogManager; + +import com.google.common.collect.ImmutableList; +import com.typesafe.config.Config; + +import javax.inject.Inject; +import javax.inject.Singleton; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import org.apache.gobblin.configuration.ConfigurationKeys; +import org.apache.gobblin.metrics.event.EventSubmitter; + + +/** + * Automatic troubleshooter will identify and prioritize the problems with the job, and display a summary to the user. + * + * Troubleshooter will collect errors & warnings from logs and combine them with various health checks. After that + * you can {@link #refineIssues()} to prioritize them and filter out noise, and then {@link #logIssueSummary()} + * to show a human-readable list of issues. + * + * Implementation and architecture notes: + * + * We convert log messages and health check results to {@link Issue}s. They will be shown to the user at the end of + * the job log. To avoid overwhelming the user, we will only collect a fixed number of issues, and will de-duplicate + * them, so that each type of problem is shown only once. + * + * Issues will be emitted in GobblinTrackingEvents at the end of the job, so that they can be collected by Gobblin + * service, and used for future platform-wide analysis. + * + * */ +@Slf4j +@Singleton +public class AutomaticTroubleshooter { + private final AutomaticTroubleshooterConfig config; + private final IssueRefinery issueRefinery; + + @Getter + private final IssueRepository issueRepository; + + private AutoTroubleshooterLogAppender troubleshooterLogger; + + @Inject + public AutomaticTroubleshooter(AutomaticTroubleshooterConfig config, IssueRepository issueRepository, + IssueRefinery issueRefinery) { + this.config = Objects.requireNonNull(config); + this.issueRepository = Objects.requireNonNull(issueRepository); + this.issueRefinery = Objects.requireNonNull(issueRefinery); + } + + /** + * Configures a troubleshooter that will be used inside Gobblin job or task. + * + * It will use small in-memory storage for issues. + * */ + public static AutomaticTroubleshooter createForJob(Config config) { + AutomaticTroubleshooterConfig troubleshooterConfig = new AutomaticTroubleshooterConfig(config); + InMemoryIssueRepository issueRepository = new InMemoryIssueRepository(); + DefaultIssueRefinery issueRefinery = new DefaultIssueRefinery(); + return new AutomaticTroubleshooter(troubleshooterConfig, issueRepository, issueRefinery); + } + + public void start() { + if (config.isDisabled()) { + logDisabledMessage(); + return; + } + setupLogAppender(); + } + + public void stop() { + if (config.isDisabled()) { + return; + } + removeLogAppender(); + } + + private void setupLogAppender() { + org.apache.log4j.Logger rootLogger = LogManager.getRootLogger(); Review comment: We should NOT use APIs of the specific Logger implementation, and instead use the SLF4j APIs. This creates all kinds of issues when you have dependents of Gobblin which are using a different logger e.g. log4j2. ########## File path: gobblin-runtime/src/main/java/org/apache/gobblin/runtime/troubleshooter/AutomaticTroubleshooterConfig.java ########## @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.runtime.troubleshooter; + +import java.util.Objects; + +import com.typesafe.config.Config; + +import javax.inject.Inject; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import org.apache.gobblin.configuration.ConfigurationKeys; +import org.apache.gobblin.util.ConfigUtils; + + +/** + * See documentation in {@link ConfigurationKeys} for explanation of settings. + * */ +@Getter +@Builder +@AllArgsConstructor +public class AutomaticTroubleshooterConfig { + private final boolean disabled; + private final boolean disableEventReporting; + + @Builder.Default + private int inMemoryRepositoryMaxSize = ConfigurationKeys.DEFAULT_TROUBLESHOOTER_IN_MEMORY_ISSUE_REPOSITORY_MAX_SIZE; + + @Inject + public AutomaticTroubleshooterConfig(Config config) { + Objects.requireNonNull(config, "Config cannot be null"); + + disabled = ConfigUtils.getBoolean(config, ConfigurationKeys.TROUBLESHOOTER_DISABLED, false); Review comment: Can we set the default to true? Let pipelines explicitly enable the trouble shooter if they desire. ########## File path: gobblin-runtime/src/main/java/org/apache/gobblin/runtime/troubleshooter/AutomaticTroubleshooterConfig.java ########## @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.runtime.troubleshooter; + +import java.util.Objects; + +import com.typesafe.config.Config; + +import javax.inject.Inject; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import org.apache.gobblin.configuration.ConfigurationKeys; +import org.apache.gobblin.util.ConfigUtils; + + +/** + * See documentation in {@link ConfigurationKeys} for explanation of settings. + * */ +@Getter +@Builder +@AllArgsConstructor +public class AutomaticTroubleshooterConfig { + private final boolean disabled; + private final boolean disableEventReporting; + + @Builder.Default + private int inMemoryRepositoryMaxSize = ConfigurationKeys.DEFAULT_TROUBLESHOOTER_IN_MEMORY_ISSUE_REPOSITORY_MAX_SIZE; + + @Inject + public AutomaticTroubleshooterConfig(Config config) { + Objects.requireNonNull(config, "Config cannot be null"); + + disabled = ConfigUtils.getBoolean(config, ConfigurationKeys.TROUBLESHOOTER_DISABLED, false); + disableEventReporting = Review comment: Same comment as above. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
