Repository: ambari Updated Branches: refs/heads/branch-1.6.0 7af4bd77a -> b95d9fddb
AMBARI-5786 - Views: Pig view parameters adjusting (Roman Rader via tbeerbower) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b95d9fdd Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b95d9fdd Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b95d9fdd Branch: refs/heads/branch-1.6.0 Commit: b95d9fddbffc4f44dbf2ca92e178f3b44b016e8a Parents: 7af4bd7 Author: tbeerbower <[email protected]> Authored: Mon May 19 10:08:55 2014 -0400 Committer: tbeerbower <[email protected]> Committed: Mon May 19 10:08:55 2014 -0400 ---------------------------------------------------------------------- .../resources/PersonalCRUDResourceManager.java | 7 +++ .../pig/resources/jobs/JobResourceManager.java | 8 +-- .../scripts/ScriptResourceManager.java | 7 +-- .../ambari/view/pig/services/BaseService.java | 11 ++-- .../view/pig/templeton/client/TempletonApi.java | 2 +- contrib/views/pig/src/main/resources/view.xml | 16 +++--- .../org/apache/ambari/view/pig/BasePigTest.java | 11 ++-- .../apache/ambari/view/pig/test/JobTest.java | 29 ++++++++++- .../apache/ambari/view/pig/test/ScriptTest.java | 2 +- .../view/pig/test/ScriptTestHDFSUnmanaged.java | 53 +++++++++++++++++++- .../view/pig/test/ScriptTestUnmanaged.java | 2 +- 11 files changed, 118 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b95d9fdd/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/PersonalCRUDResourceManager.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/PersonalCRUDResourceManager.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/PersonalCRUDResourceManager.java index 650d9a2..b54ca4b 100644 --- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/PersonalCRUDResourceManager.java +++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/PersonalCRUDResourceManager.java @@ -87,4 +87,11 @@ public class PersonalCRUDResourceManager<T extends PersonalResource> extends CRU } return result; } + + protected String getUsername() { + String userName = context.getProperties().get("dataworker.username"); + if (userName == null) + userName = context.getUsername(); + return userName; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/b95d9fdd/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java index abc2ddd..2d6c198 100644 --- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java +++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java @@ -64,7 +64,7 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> { */ public TempletonApi getTempletonApi() { if (api == null) { - api = new TempletonApi(context.getProperties().get("dataworker.templeton_url"), + api = new TempletonApi(context.getProperties().get("dataworker.webhcat.url"), getTempletonUser(), getTempletonUser(), context); } return api; @@ -135,8 +135,8 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> { */ private void submitJob(PigJob job) { String date = new SimpleDateFormat("dd-MM-yyyy-HH-mm-ss").format(new Date()); - String statusdir = String.format(context.getProperties().get("dataworker.pigJobsPath") + - "/%s/%s_%s", getTempletonUser(), + String statusdir = String.format(context.getProperties().get("dataworker.jobs.path") + + "/%s/%s_%s", getUsername(), job.getTitle().toLowerCase().replaceAll("[^a-zA-Z0-9 ]+", "").replace(" ", "_"), date); @@ -295,6 +295,6 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> { * @return username in templeton */ private String getTempletonUser() { - return context.getProperties().get("dataworker.templeton_user"); + return context.getProperties().get("dataworker.webhcat.user"); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/b95d9fdd/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/scripts/ScriptResourceManager.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/scripts/ScriptResourceManager.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/scripts/ScriptResourceManager.java index f98bdab..9ecf30a 100644 --- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/scripts/ScriptResourceManager.java +++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/scripts/ScriptResourceManager.java @@ -58,9 +58,9 @@ public class ScriptResourceManager extends PersonalCRUDResourceManager<PigScript } private void createDefaultScriptFile(PigScript object) { - String userScriptsPath = context.getProperties().get("dataworker.userScriptsPath"); + String userScriptsPath = context.getProperties().get("dataworker.scripts.path"); if (userScriptsPath == null) { - String msg = "dataworker.userScriptsPath is not configured!"; + String msg = "dataworker.scripts.path is not configured!"; LOG.error(msg); throw new WebServiceException(msg); } @@ -72,7 +72,7 @@ public class ScriptResourceManager extends PersonalCRUDResourceManager<PigScript String normalizedName = object.getTitle().replaceAll("[^a-zA-Z0-9 ]+", "").replaceAll(" ", "_").toLowerCase(); String timestamp = new SimpleDateFormat("yyyy-MM-dd_hh-mm").format(new Date()); newFilePath = String.format(userScriptsPath + - "/%s/%s-%s%s.pig", context.getUsername(), + "/%s/%s-%s%s.pig", getUsername(), normalizedName, timestamp, (checkId == 0)?"":"_"+checkId); LOG.debug("Trying to create new file " + newFilePath); @@ -105,4 +105,5 @@ public class ScriptResourceManager extends PersonalCRUDResourceManager<PigScript object.setPigScript(newFilePath); getPigStorage().store(object); } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/b95d9fdd/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/services/BaseService.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/services/BaseService.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/services/BaseService.java index 06e00c2..4c25be7 100644 --- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/services/BaseService.java +++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/services/BaseService.java @@ -69,8 +69,6 @@ public class BaseService { if (hdfsApi == null) { Thread.currentThread().setContextClassLoader(null); - String userName = context.getUsername(); - String defaultFS = context.getProperties().get("dataworker.defaultFs"); if (defaultFS == null) { String message = "dataworker.defaultFs is not configured!"; @@ -79,7 +77,7 @@ public class BaseService { } try { - hdfsApi = new HdfsApi(defaultFS, userName); + hdfsApi = new HdfsApi(defaultFS, getHdfsUsername(context)); LOG.info("HdfsApi connected OK"); } catch (IOException e) { String message = "HdfsApi IO error: " + e.getMessage(); @@ -94,6 +92,13 @@ public class BaseService { return hdfsApi; } + public static String getHdfsUsername(ViewContext context) { + String userName = context.getProperties().get("dataworker.hdfs.username"); + if (userName == null) + userName = context.getUsername(); + return userName; + } + protected HdfsApi getHdfsApi() { return getHdfsApi(context); } http://git-wip-us.apache.org/repos/asf/ambari/blob/b95d9fdd/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonApi.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonApi.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonApi.java index 4fe61cd..9ad54ac 100644 --- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonApi.java +++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonApi.java @@ -52,7 +52,7 @@ public class TempletonApi { /** * TempletonApi constructor - * @param api dataworker.templeton_url + * @param api dataworker.webhcat.url * @param username templeton username * @param doAs doAs argument * @param context context with URLStreamProvider http://git-wip-us.apache.org/repos/asf/ambari/blob/b95d9fdd/contrib/views/pig/src/main/resources/view.xml ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/main/resources/view.xml b/contrib/views/pig/src/main/resources/view.xml index f6c6c30..7f59758 100644 --- a/contrib/views/pig/src/main/resources/view.xml +++ b/contrib/views/pig/src/main/resources/view.xml @@ -27,24 +27,24 @@ </parameter> <parameter> - <name>dataworker.hdfs_user</name> + <name>dataworker.hdfs.username</name> <description>HDFS user.name and doAs</description> <required>false</required> </parameter> <parameter> - <name>dataworker.use_current_user</name> - <description>If true, get user from Ambari auth</description> - <required>true</required> + <name>dataworker.username</name> + <description>The username (defaults to ViewContext username)</description> + <required>false</required> </parameter> <parameter> - <name>dataworker.userScriptsPath</name> + <name>dataworker.scripts.path</name> <description>Directory to store scripts</description> <required>true</required> </parameter> <parameter> - <name>dataworker.pigJobsPath</name> + <name>dataworker.jobs.path</name> <description>Directory to store jobs (for templeton status dir)</description> <required>true</required> </parameter> @@ -58,13 +58,13 @@ </parameter> <parameter> - <name>dataworker.templeton_url</name> + <name>dataworker.webhcat.url</name> <description>Templeton url</description> <required>true</required> </parameter> <parameter> - <name>dataworker.templeton_user</name> + <name>dataworker.webhcat.user</name> <description>Templeton user.name and doAs</description> <required>true</required> </parameter> http://git-wip-us.apache.org/repos/asf/ambari/blob/b95d9fdd/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/BasePigTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/BasePigTest.java b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/BasePigTest.java index 7352157..ab158a3 100644 --- a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/BasePigTest.java +++ b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/BasePigTest.java @@ -37,6 +37,7 @@ public abstract class BasePigTest { protected ViewContext context; protected static File pigStorageFile; protected static File baseDir; + protected Map<String, String> properties; protected static String DATA_DIRECTORY = "./target/PigTest"; @@ -52,17 +53,17 @@ public abstract class BasePigTest { handler = createNiceMock(ViewResourceHandler.class); context = createNiceMock(ViewContext.class); - Map<String, String> properties = new HashMap<String, String>(); + properties = new HashMap<String, String>(); baseDir = new File(DATA_DIRECTORY) .getAbsoluteFile(); pigStorageFile = new File("./target/BasePigTest/storage.dat") .getAbsoluteFile(); properties.put("dataworker.storagePath", pigStorageFile.toString()); - properties.put("dataworker.templeton_url", "localhost:50111/templeton/v1"); - properties.put("dataworker.templeton_user", "admin"); - properties.put("dataworker.userScriptsPath", "/tmp/.pigscripts"); - properties.put("dataworker.pigJobsPath", "/tmp/.pigjobs"); + properties.put("dataworker.webhcat.url", "localhost:50111/templeton/v1"); + properties.put("dataworker.webhcat.user", "admin"); + properties.put("dataworker.scripts.path", "/tmp/.pigscripts"); + properties.put("dataworker.jobs.path", "/tmp/.pigjobs"); setupProperties(properties, baseDir); http://git-wip-us.apache.org/repos/asf/ambari/blob/b95d9fdd/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/JobTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/JobTest.java b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/JobTest.java index 6cf4bd7..2785e95 100644 --- a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/JobTest.java +++ b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/JobTest.java @@ -106,7 +106,7 @@ public class JobTest extends BasePigTest { Assert.assertTrue(obj.containsKey("job")); Assert.assertNotNull(((PigJob) obj.get("job")).getId()); Assert.assertFalse(((PigJob) obj.get("job")).getId().isEmpty()); - Assert.assertTrue(((PigJob) obj.get("job")).getStatusDir().startsWith("/tmp/.pigjobs/admin/test")); + Assert.assertTrue(((PigJob) obj.get("job")).getStatusDir().startsWith("/tmp/.pigjobs/ambari-qa/test")); PigJob job = ((PigJob) obj.get("job")); Assert.assertEquals(PigJob.Status.SUBMITTED, job.getStatus()); @@ -114,6 +114,31 @@ public class JobTest extends BasePigTest { } @Test + public void testSubmitJobUsernameProvided() throws Exception { + HdfsApi hdfsApi = createNiceMock(HdfsApi.class); + expect(hdfsApi.copy(eq("/tmp/script.pig"), startsWith("/tmp/.pigjobs/"))).andReturn(true); + + ByteArrayOutputStream do_stream = new ByteArrayOutputStream(); + + FSDataOutputStream stream = new FSDataOutputStream(do_stream); + expect(hdfsApi.create(anyString(), eq(true))).andReturn(stream); + replay(hdfsApi); + JobService.setHdfsApi(hdfsApi); + + TempletonApi api = createNiceMock(TempletonApi.class); + jobService.getResourceManager().setTempletonApi(api); + TempletonApi.JobData data = api.new JobData(); + expect(api.runPigQuery((File) anyObject(), anyString(), eq("-useHCatalog"))).andReturn(data); + replay(api); + + properties.put("dataworker.username", "luke"); + Response response = doCreateJob("Test", "/tmp/script.pig", "-useHCatalog"); + JSONObject obj = (JSONObject)response.getEntity(); + Assert.assertTrue(obj.containsKey("job")); + Assert.assertTrue(((PigJob) obj.get("job")).getStatusDir().startsWith("/tmp/.pigjobs/luke/test")); + } + + @Test public void testSubmitJobNoArguments() throws Exception { HdfsApi hdfsApi = createNiceMock(HdfsApi.class); expect(hdfsApi.copy(eq("/tmp/script.pig"), startsWith("/tmp/.pigjobs/"))).andReturn(true); @@ -140,7 +165,7 @@ public class JobTest extends BasePigTest { Assert.assertTrue(obj.containsKey("job")); Assert.assertNotNull(((PigJob) obj.get("job")).getId()); Assert.assertFalse(((PigJob) obj.get("job")).getId().isEmpty()); - Assert.assertTrue(((PigJob) obj.get("job")).getStatusDir().startsWith("/tmp/.pigjobs/admin/test")); + Assert.assertTrue(((PigJob) obj.get("job")).getStatusDir().startsWith("/tmp/.pigjobs/ambari-qa/test")); PigJob job = ((PigJob) obj.get("job")); Assert.assertEquals(PigJob.Status.SUBMITTED, job.getStatus()); http://git-wip-us.apache.org/repos/asf/ambari/blob/b95d9fdd/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTest.java b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTest.java index e7ddf3d..caba2cb 100644 --- a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTest.java +++ b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTest.java @@ -60,7 +60,7 @@ public class ScriptTest extends HDFSTest { @Override protected void setupProperties(Map<String, String> properties, File baseDir) throws Exception { super.setupProperties(properties, baseDir); - properties.put("dataworker.userScriptsPath", "/tmp/.pigscripts"); + properties.put("dataworker.scripts.path", "/tmp/.pigscripts"); } private Response doCreateScript() { http://git-wip-us.apache.org/repos/asf/ambari/blob/b95d9fdd/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestHDFSUnmanaged.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestHDFSUnmanaged.java b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestHDFSUnmanaged.java index 9eacbee..93731f7 100644 --- a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestHDFSUnmanaged.java +++ b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestHDFSUnmanaged.java @@ -27,6 +27,9 @@ import org.apache.ambari.view.pig.persistence.Storage; import org.apache.ambari.view.pig.resources.files.FileService; import org.apache.ambari.view.pig.resources.scripts.ScriptService; import org.apache.ambari.view.pig.persistence.utils.StorageUtil; +import org.apache.ambari.view.pig.resources.scripts.models.PigScript; +import org.apache.ambari.view.pig.services.BaseService; +import org.json.simple.JSONObject; import org.junit.*; import javax.ws.rs.core.Response; @@ -70,7 +73,7 @@ public class ScriptTestHDFSUnmanaged extends HDFSTest { .getAbsoluteFile(); properties.put("dataworker.storagePath", pigStorageFile.toString()); -// properties.put("dataworker.userScriptsPath", "/tmp/.pigscripts"); +// properties.put("dataworker.scripts.path", "/tmp/.pigscripts"); properties.put("dataworker.defaultFs", hdfsURI); expect(context.getProperties()).andReturn(properties).anyTimes(); @@ -83,6 +86,34 @@ public class ScriptTestHDFSUnmanaged extends HDFSTest { } @Test + public void createScriptAutoCreateUsername() throws IOException, InterruptedException { + Map<String, String> properties = new HashMap<String, String>(); + baseDir = new File(DATA_DIRECTORY) + .getAbsoluteFile(); + pigStorageFile = new File("./target/BasePigTest/storage.dat") + .getAbsoluteFile(); + + properties.put("dataworker.storagePath", pigStorageFile.toString()); + properties.put("dataworker.scripts.path", "/tmp/.pigscripts"); + properties.put("dataworker.defaultFs", hdfsURI); + + expect(context.getProperties()).andReturn(properties).anyTimes(); + expect(context.getUsername()).andReturn("ambari-qa").anyTimes(); + + replay(handler, context); + scriptService = getService(ScriptService.class, handler, context); + + Response createdScript = doCreateScript("Test", null); + String createdScriptPath = ((PigScript) ((JSONObject) createdScript.getEntity()).get("script")).getPigScript(); + Assert.assertTrue(createdScriptPath.startsWith("/tmp/.pigscripts/ambari-qa/")); + + properties.put("dataworker.username", "luke"); + Response createdScript2 = doCreateScript("Test", null); + String createdScriptPath2 = ((PigScript) ((JSONObject) createdScript2.getEntity()).get("script")).getPigScript(); + Assert.assertTrue(createdScriptPath2.startsWith("/tmp/.pigscripts/luke/")); + } + + @Test public void createScriptAutoCreateNoStoragePath() throws IOException, InterruptedException { Map<String, String> properties = new HashMap<String, String>(); baseDir = new File(DATA_DIRECTORY) @@ -91,7 +122,7 @@ public class ScriptTestHDFSUnmanaged extends HDFSTest { .getAbsoluteFile(); // properties.put("dataworker.storagePath", pigStorageFile.toString()); - properties.put("dataworker.userScriptsPath", "/tmp/.pigscripts"); + properties.put("dataworker.scripts.path", "/tmp/.pigscripts"); properties.put("dataworker.defaultFs", hdfsURI); expect(context.getProperties()).andReturn(properties).anyTimes(); @@ -103,6 +134,24 @@ public class ScriptTestHDFSUnmanaged extends HDFSTest { Assert.assertEquals(InstanceKeyValueStorage.class.getSimpleName(), storage.getClass().getSimpleName()); } + @Test + public void hdfsApiNoUsernameProvided() throws IOException, InterruptedException { + Map<String, String> properties = new HashMap<String, String>(); + properties.put("dataworker.defaultFs", hdfsURI); + + expect(context.getProperties()).andReturn(properties).anyTimes(); + expect(context.getUsername()).andReturn("ambari-qa").anyTimes(); + + replay(context); + + // no dataworker.hdfs.username property + Assert.assertEquals("ambari-qa", BaseService.getHdfsUsername(context)); + + // with dataworker.hdfs.username property + properties.put("dataworker.hdfs.username", "luke"); + Assert.assertEquals("luke", BaseService.getHdfsUsername(context)); + } + private Response doCreateScript(String title, String path) { return ScriptTest.doCreateScript(title, path, scriptService); } http://git-wip-us.apache.org/repos/asf/ambari/blob/b95d9fdd/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestUnmanaged.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestUnmanaged.java b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestUnmanaged.java index a6138b5..8b1db4a 100644 --- a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestUnmanaged.java +++ b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestUnmanaged.java @@ -79,7 +79,7 @@ public class ScriptTestUnmanaged extends BasePigTest { public void createScriptAutoCreateNoDefaultFS() { Map<String, String> properties = new HashMap<String, String>(); properties.put("dataworker.storagePath", pigStorageFile.toString()); - properties.put("dataworker.userScriptsPath", "/tmp/.pigscripts"); + properties.put("dataworker.scripts.path", "/tmp/.pigscripts"); expect(context.getProperties()).andReturn(properties).anyTimes(); expect(context.getUsername()).andReturn("ambari-qa").anyTimes();
