Repository: ambari Updated Branches: refs/heads/trunk 88b170d49 -> a2086b1fc
AMBARI-6533 - Add API to get file log location in the task execution output, currently hardcoded in UI (Alejandro Fernandez via jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a2086b1f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a2086b1f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a2086b1f Branch: refs/heads/trunk Commit: a2086b1fc88ccc85b9765a58d53f48f6c14d8ce1 Parents: 88b170d Author: Jonathan Hurley <[email protected]> Authored: Mon Aug 11 14:39:36 2014 -0400 Committer: Jonathan Hurley <[email protected]> Committed: Mon Aug 11 15:08:37 2014 -0400 ---------------------------------------------------------------------- .../apache/ambari/server/orm/dao/DaoUtils.java | 2 +- .../server/orm/dao/HostRoleCommandDAO.java | 1 - .../orm/entities/HostRoleCommandEntity_.java | 51 +++++++++++++ .../server/upgrade/UpgradeCatalog170.java | 80 +++++++++++++++++--- .../server/upgrade/UpgradeCatalog170Test.java | 74 ++++++++++++------ 5 files changed, 173 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a2086b1f/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/DaoUtils.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/DaoUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/DaoUtils.java index b4e29b9..d92491f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/DaoUtils.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/DaoUtils.java @@ -31,7 +31,7 @@ import java.util.Collections; import java.util.List; @Singleton -class DaoUtils { +public class DaoUtils { public <T> List<T> selectAll(EntityManager entityManager, Class<T> entityClass) { CriteriaBuilder criteriaBuilder = entityManager.getCriteriaBuilder(); http://git-wip-us.apache.org/repos/asf/ambari/blob/a2086b1f/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java index 71c27cb..56dd304 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java @@ -266,5 +266,4 @@ public class HostRoleCommandDAO { public void removeByPK(int taskId) { remove(findByPK(taskId)); } - } http://git-wip-us.apache.org/repos/asf/ambari/blob/a2086b1f/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity_.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity_.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity_.java new file mode 100644 index 0000000..4dad21a --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity_.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.orm.entities; + +import javax.persistence.metamodel.SingularAttribute; + + +/** + * This class exists so that JPQL can use static singular attributes that are strongly typed + * as opposed to Java reflection like HostRoleCommandEntity.get("fieldname") + */ [email protected](HostRoleCommandEntity.class) +public class HostRoleCommandEntity_ { + public static volatile SingularAttribute<HostRoleCommandEntity, Long> taskId; + public static volatile SingularAttribute<HostRoleCommandEntity, Long> requestId; + public static volatile SingularAttribute<HostRoleCommandEntity, Long> stageId; + public static volatile SingularAttribute<HostRoleCommandEntity, String> hostName; + public static volatile SingularAttribute<HostRoleCommandEntity, String> role; + public static volatile SingularAttribute<HostRoleCommandEntity, String> event; + public static volatile SingularAttribute<HostRoleCommandEntity, Integer> exitcode; + public static volatile SingularAttribute<HostRoleCommandEntity, String> status; + public static volatile SingularAttribute<HostRoleCommandEntity, byte[]> stdError; + public static volatile SingularAttribute<HostRoleCommandEntity, byte[]> stdOut; + public static volatile SingularAttribute<HostRoleCommandEntity, String> outputLog; + public static volatile SingularAttribute<HostRoleCommandEntity, String> errorLog; + public static volatile SingularAttribute<HostRoleCommandEntity, byte[]> structuredOut; + public static volatile SingularAttribute<HostRoleCommandEntity, Long> startTime; + public static volatile SingularAttribute<HostRoleCommandEntity, Long> endTime; + public static volatile SingularAttribute<HostRoleCommandEntity, Long> lastAttemptTime; + public static volatile SingularAttribute<HostRoleCommandEntity, Short> attemptCount; + public static volatile SingularAttribute<HostRoleCommandEntity, String> roleCommand; + public static volatile SingularAttribute<HostRoleCommandEntity, String> commandDetail; + public static volatile SingularAttribute<HostRoleCommandEntity, String> customCommandName; +} + http://git-wip-us.apache.org/repos/asf/ambari/blob/a2086b1f/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java index 31123ce..758efe9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java @@ -28,9 +28,14 @@ import java.util.Map.Entry; import java.util.Set; import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.orm.DBAccessor.DBColumnInfo; +import org.apache.ambari.server.orm.dao.DaoUtils; +import org.apache.ambari.server.orm.dao.HostRoleCommandDAO; +import org.apache.ambari.server.orm.entities.HostRoleCommandEntity; +import org.apache.ambari.server.orm.entities.HostRoleCommandEntity_; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; @@ -41,6 +46,10 @@ import org.slf4j.LoggerFactory; import com.google.inject.Inject; import com.google.inject.Injector; +import javax.persistence.EntityManager; +import javax.persistence.TypedQuery; +import javax.persistence.criteria.*; + /** * Upgrade catalog for version 1.7.0. */ @@ -82,6 +91,8 @@ public class UpgradeCatalog170 extends AbstractUpgradeCatalog { this.injector = injector; } + @Inject + DaoUtils daoUtils; // ----- AbstractUpgradeCatalog -------------------------------------------- @@ -194,17 +205,6 @@ public class UpgradeCatalog170 extends AbstractUpgradeCatalog { dbAccessor.addColumn("host_role_command", new DBColumnInfo("error_log", String.class, 255, null, true)); - // Update historic records with the log paths, but only enough so as to not prolong the upgrade process - if (dbType.equals(Configuration.POSTGRES_DB_NAME) || dbType.equals(Configuration.ORACLE_DB_NAME)) { - // Postgres and Oracle use a different concatenation operator. - dbAccessor.executeQuery("UPDATE host_role_command SET output_log = ('/var/lib/ambari-agent/data/output-' || CAST(task_id AS VARCHAR(20)) || '.txt') WHERE task_id IN (SELECT task_id FROM host_role_command WHERE output_log IS NULL OR output_log = '' ORDER BY task_id DESC LIMIT 1000);"); - dbAccessor.executeQuery("UPDATE host_role_command SET error_log = ('/var/lib/ambari-agent/data/errors-' || CAST(task_id AS VARCHAR(20)) || '.txt') WHERE task_id IN (SELECT task_id FROM host_role_command WHERE error_log IS NULL OR error_log = '' ORDER BY task_id DESC LIMIT 1000);"); - } else if (dbType.equals(Configuration.MYSQL_DB_NAME)) { - // MySQL uses a different concatenation operator. - dbAccessor.executeQuery("UPDATE host_role_command SET output_log = CONCAT('/var/lib/ambari-agent/data/output-', task_id, '.txt') WHERE task_id IN (SELECT task_id FROM host_role_command WHERE output_log IS NULL OR output_log = '' ORDER BY task_id DESC LIMIT 1000);"); - dbAccessor.executeQuery("UPDATE host_role_command SET error_log = CONCAT('/var/lib/ambari-agent/data/errors-', task_id, '.txt') WHERE task_id IN (SELECT task_id FROM host_role_command WHERE error_log IS NULL OR error_log = '' ORDER BY task_id DESC LIMIT 1000);"); - } - addAlertingFrameworkDDL(); //service config versions changes @@ -333,6 +333,64 @@ public class UpgradeCatalog170 extends AbstractUpgradeCatalog { dbAccessor.executeQuery("INSERT INTO ambari_sequences(sequence_name, " + valueColumnName + ") " + "VALUES('alert_current_id_seq', 0)", false); + + // Update historic records with the log paths, but only enough so as to not prolong the upgrade process + executeInTransaction(new Runnable() { + @Override + public void run() { + try { + HostRoleCommandDAO hostRoleCommandDAO = injector.getInstance(HostRoleCommandDAO.class); + EntityManager em = getEntityManagerProvider().get(); + CriteriaBuilder cb = em.getCriteriaBuilder(); + CriteriaQuery<HostRoleCommandEntity> cq1 = cb.createQuery(HostRoleCommandEntity.class); + CriteriaQuery<HostRoleCommandEntity> cq2 = cb.createQuery(HostRoleCommandEntity.class); + Root<HostRoleCommandEntity> hrc1 = cq1.from(HostRoleCommandEntity.class); + Root<HostRoleCommandEntity> hrc2 = cq1.from(HostRoleCommandEntity.class); + + // Rather than using Java reflection, which is more susceptible to breaking, use the classname_.field canonical model + // that is safer because it exposes the persistent attributes statically. + Expression<Long> taskID1 = hrc1.get(HostRoleCommandEntity_.taskId); + Expression<Long> taskID2 = hrc2.get(HostRoleCommandEntity_.taskId); + Expression<String> outputLog = hrc1.get(HostRoleCommandEntity_.outputLog); + Expression<String> errorLog = hrc2.get(HostRoleCommandEntity_.errorLog); + + Predicate p1 = cb.isNull(outputLog); + Predicate p2 = cb.equal(outputLog, ""); + Predicate p1_or_2 = cb.or(p1, p2); + + Predicate p3 = cb.isNull(errorLog); + Predicate p4 = cb.equal(errorLog, ""); + Predicate p3_or_4 = cb.or(p3, p4); + + if (daoUtils == null) { + daoUtils = new DaoUtils(); + } + + // Update output_log + cq1.select(hrc1).where(p1_or_2).orderBy(cb.desc(taskID1)); + TypedQuery<HostRoleCommandEntity> q1 = em.createQuery(cq1); + q1.setMaxResults(1000); + List<HostRoleCommandEntity> r1 = daoUtils.selectList(q1); + for (HostRoleCommandEntity entity : r1) { + entity.setOutputLog("/var/lib/ambari-agent/data/output-" + entity.getTaskId() + ".txt"); + hostRoleCommandDAO.merge(entity); + } + + // Update error_log + cq2.select(hrc2).where(p3_or_4).orderBy(cb.desc(taskID2)); + TypedQuery<HostRoleCommandEntity> q2 = em.createQuery(cq2); + q2.setMaxResults(1000); + List<HostRoleCommandEntity> r2 = daoUtils.selectList(q2); + for (HostRoleCommandEntity entity : r2) { + entity.setErrorLog("/var/lib/ambari-agent/data/errors-" + entity.getTaskId() + ".txt"); + hostRoleCommandDAO.merge(entity); + } + } catch (Exception e) { + LOG.warn("Could not populate historic records with output_log and error_log in host_role_command table. ", e); + } + } + }); + moveGlobalsToEnv(); addEnvContentFields(); addMissingConfigs(); http://git-wip-us.apache.org/repos/asf/ambari/blob/a2086b1f/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java index a0f1e65..63b64e4 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java @@ -21,31 +21,19 @@ package org.apache.ambari.server.upgrade; import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertNull; import static junit.framework.Assert.assertTrue; -import static org.easymock.EasyMock.anyObject; -import static org.easymock.EasyMock.capture; -import static org.easymock.EasyMock.createMockBuilder; -import static org.easymock.EasyMock.createNiceMock; -import static org.easymock.EasyMock.createStrictMock; -import static org.easymock.EasyMock.eq; -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.expectLastCall; -import static org.easymock.EasyMock.replay; -import static org.easymock.EasyMock.verify; +import static org.easymock.EasyMock.*; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.orm.DBAccessor; +import org.apache.ambari.server.orm.dao.DaoUtils; +import org.apache.ambari.server.orm.entities.HostRoleCommandEntity; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; @@ -53,18 +41,31 @@ import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.StackId; import org.easymock.Capture; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; -import com.google.inject.Binder; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Module; +import com.google.inject.*; +import javax.persistence.EntityManager; +import javax.persistence.EntityTransaction; +import javax.persistence.TypedQuery; +import javax.persistence.criteria.*; +import javax.persistence.metamodel.SingularAttribute; /** * UpgradeCatalog170 unit tests. */ public class UpgradeCatalog170Test { + Provider<EntityManager> entityManagerProvider = createStrictMock(Provider.class); + EntityManager entityManager = createStrictMock(EntityManager.class); + + @Before + public void init() { + reset(entityManagerProvider); + expect(entityManagerProvider.get()).andReturn(entityManager).anyTimes(); + replay(entityManagerProvider); + } + @Test public void testExecuteDDLUpdates() throws Exception { @@ -155,11 +156,23 @@ public class UpgradeCatalog170Test { Clusters clusters = createStrictMock(Clusters.class); Config config = createStrictMock(Config.class); + EntityTransaction trans = createNiceMock(EntityTransaction.class); + CriteriaBuilder cb = createNiceMock(CriteriaBuilder.class); + CriteriaQuery<HostRoleCommandEntity> cq = createNiceMock(CriteriaQuery.class); + Root<HostRoleCommandEntity> hrc = (Root<HostRoleCommandEntity>) createNiceMock(Root.class); + Path<Long> taskId = null; + Path<String> outputLog = null; + Path<String> errorLog = null; + Order o = createNiceMock(Order.class); + TypedQuery<HostRoleCommandEntity> q = createNiceMock(TypedQuery.class); + List<HostRoleCommandEntity> r = new ArrayList<HostRoleCommandEntity>(); + Method m = AbstractUpgradeCatalog.class.getDeclaredMethod ("updateConfigurationProperties", String.class, Map.class, boolean.class, boolean.class); + Method n = AbstractUpgradeCatalog.class.getDeclaredMethod("getEntityManagerProvider"); UpgradeCatalog170 upgradeCatalog = createMockBuilder(UpgradeCatalog170.class) - .addMockedMethod(m).createMock(); + .addMockedMethod(m).addMockedMethod(n).createMock(); Map<String, Cluster> clustersMap = new HashMap<String, Cluster>(); clustersMap.put("c1", cluster); @@ -191,6 +204,22 @@ public class UpgradeCatalog170Test { Collections.singletonMap("hbase_regionserver_xmn_ratio", "0.2"), false, false); expectLastCall(); + expect(entityManager.getTransaction()).andReturn(trans).anyTimes(); + expect(entityManager.getCriteriaBuilder()).andReturn(cb).anyTimes(); + expect(entityManager.createQuery(cq)).andReturn(q).anyTimes(); + expect(trans.isActive()).andReturn(true).anyTimes(); + expect(upgradeCatalog.getEntityManagerProvider()).andReturn(entityManagerProvider).anyTimes(); + expect(cb.createQuery(HostRoleCommandEntity.class)).andReturn(cq).anyTimes(); + expect(cb.desc(taskId)).andReturn(o).anyTimes(); + expect(cq.from(HostRoleCommandEntity.class)).andReturn(hrc).anyTimes(); + expect(cq.select(hrc)).andReturn(cq).anyTimes(); + expect(cq.where(anyObject(Predicate.class))).andReturn(cq).anyTimes(); + expect(hrc.get(isA(SingularAttribute.class))).andReturn(taskId).times(2); + expect(hrc.get(isA(SingularAttribute.class))).andReturn(outputLog).once(); + expect(hrc.get(isA(SingularAttribute.class))).andReturn(errorLog).once(); + expect(q.setMaxResults(1000)).andReturn(q).anyTimes(); + expect(q.getResultList()).andReturn(r).anyTimes(); + expect(configuration.getDatabaseUrl()).andReturn(Configuration.JDBC_IN_MEMORY_URL).anyTimes(); expect(injector.getInstance(ConfigHelper.class)).andReturn(configHelper).anyTimes(); expect(injector.getInstance(AmbariManagementController.class)).andReturn(amc).anyTimes(); @@ -204,7 +233,8 @@ public class UpgradeCatalog170Test { expect(configHelper.findConfigTypesByPropertyName(new StackId("HDP", "2.1"), "content")).andReturn(envDicts).once(); expect(configHelper.getPropertyValueFromStackDefenitions(cluster, "hadoop-env", "content")).andReturn("env file contents").once(); - replay(upgradeCatalog, dbAccessor, configuration, injector, cluster, clusters, amc, config, configHelper); + replay(entityManager, trans, upgradeCatalog, cb, cq, hrc, q); + replay(dbAccessor, configuration, injector, cluster, clusters, amc, config, configHelper); Class<?> c = AbstractUpgradeCatalog.class; Field f = c.getDeclaredField("configuration");
