Repository: incubator-nifi Updated Branches: refs/heads/develop 953d9e59a -> c14b2cebf
NIFI-271 Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/c14b2ceb Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/c14b2ceb Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/c14b2ceb Branch: refs/heads/develop Commit: c14b2cebf8e28d898c39acd661e5923f81c1e5e9 Parents: 953d9e5 Author: joewitt <[email protected]> Authored: Sat Apr 25 08:27:24 2015 -0400 Committer: joewitt <[email protected]> Committed: Sat Apr 25 08:27:24 2015 -0400 ---------------------------------------------------------------------- .../nifi-yandex-processors/pom.xml | 4 +- .../nifi/processors/yandex/YandexTranslate.java | 381 +++++++++---------- .../processors/yandex/model/Translation.java | 55 +-- .../nifi/processors/yandex/util/Languages.java | 124 +++--- .../processors/yandex/TestYandexTranslate.java | 317 ++++++++------- .../nifi-language-translation-bundle/pom.xml | 28 +- 6 files changed, 453 insertions(+), 456 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/c14b2ceb/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/pom.xml ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/pom.xml b/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/pom.xml index a5f9f0e..03776ca 100644 --- a/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/pom.xml +++ b/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/pom.xml @@ -36,8 +36,8 @@ </dependency> <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-client</artifactId> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-client</artifactId> </dependency> <dependency> <groupId>com.sun.jersey</groupId> http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/c14b2ceb/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/main/java/org/apache/nifi/processors/yandex/YandexTranslate.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/main/java/org/apache/nifi/processors/yandex/YandexTranslate.java b/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/main/java/org/apache/nifi/processors/yandex/YandexTranslate.java index bd83a8a..8398152 100644 --- a/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/main/java/org/apache/nifi/processors/yandex/YandexTranslate.java +++ b/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/main/java/org/apache/nifi/processors/yandex/YandexTranslate.java @@ -73,76 +73,76 @@ import com.sun.jersey.core.util.MultivaluedMapImpl; @Tags({"yandex", "translate", "translation", "language"}) @CapabilityDescription("Translates content and attributes from one language to another") @WritesAttributes({ - @WritesAttribute(attribute="yandex.translate.failure.reason", description="If the text cannot be translated, this attribute will be set indicating the reason for the failure"), - @WritesAttribute(attribute="language", description="When the translation succeeds, if the content was translated, this attribute will be set indicating the new language of the content") + @WritesAttribute(attribute = "yandex.translate.failure.reason", description = "If the text cannot be translated, this attribute will be set indicating the reason for the failure"), + @WritesAttribute(attribute = "language", description = "When the translation succeeds, if the content was translated, this attribute will be set indicating the new language of the content") }) -@DynamicProperty(name="The name of an attribute to set that will contain the translated text of the value", - value="The value to translate", - supportsExpressionLanguage=true, - description="User-defined properties are used to translate arbitrary text based on attributes.") +@DynamicProperty(name = "The name of an attribute to set that will contain the translated text of the value", + value = "The value to translate", + supportsExpressionLanguage = true, + description = "User-defined properties are used to translate arbitrary text based on attributes.") public class YandexTranslate extends AbstractProcessor { - + public static final PropertyDescriptor KEY = new PropertyDescriptor.Builder() - .name("Yandex API Key") - .description("The API Key that is registered with Yandex") - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .required(true) - .build(); + .name("Yandex API Key") + .description("The API Key that is registered with Yandex") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .required(true) + .build(); public static final PropertyDescriptor SOURCE_LANGUAGE = new PropertyDescriptor.Builder() - .name("Input Language") - .description("The language of incoming data") - .required(true) - .defaultValue("es") - .expressionLanguageSupported(true) - .addValidator(new LanguageNameValidator()) - .build(); + .name("Input Language") + .description("The language of incoming data") + .required(true) + .defaultValue("es") + .expressionLanguageSupported(true) + .addValidator(new LanguageNameValidator()) + .build(); public static final PropertyDescriptor TARGET_LANGUAGE = new PropertyDescriptor.Builder() - .name("Target Language") - .description("The language to translate the text into") - .required(true) - .defaultValue("en") - .expressionLanguageSupported(true) - .addValidator(new LanguageNameValidator()) - .build(); + .name("Target Language") + .description("The language to translate the text into") + .required(true) + .defaultValue("en") + .expressionLanguageSupported(true) + .addValidator(new LanguageNameValidator()) + .build(); public static final PropertyDescriptor TRANSLATE_CONTENT = new PropertyDescriptor.Builder() - .name("Translate Content") - .description("Specifies whether or not the content should be translated. If false, only the text specified by user-defined properties will be translated.") - .required(true) - .allowableValues("true", "false") - .defaultValue("false") - .build(); + .name("Translate Content") + .description("Specifies whether or not the content should be translated. If false, only the text specified by user-defined properties will be translated.") + .required(true) + .allowableValues("true", "false") + .defaultValue("false") + .build(); public static final PropertyDescriptor CHARACTER_SET = new PropertyDescriptor.Builder() - .name("Character Set") - .description("Specifies the character set of the data to be translated") - .required(true) - .defaultValue("UTF-8") - .expressionLanguageSupported(true) - .addValidator(StandardValidators.CHARACTER_SET_VALIDATOR) - .build(); + .name("Character Set") + .description("Specifies the character set of the data to be translated") + .required(true) + .defaultValue("UTF-8") + .expressionLanguageSupported(true) + .addValidator(StandardValidators.CHARACTER_SET_VALIDATOR) + .build(); public static final Relationship REL_SUCCESS = new Relationship.Builder() - .name("success") - .description("This relationship is used when the translation is successful") - .build(); + .name("success") + .description("This relationship is used when the translation is successful") + .build(); public static final Relationship REL_COMMS_FAILURE = new Relationship.Builder() - .name("comms.failure") - .description("This relationship is used when the translation fails due to a problem such as a network failure, and for which the translation should be attempted again") - .build(); + .name("comms.failure") + .description("This relationship is used when the translation fails due to a problem such as a network failure, and for which the translation should be attempted again") + .build(); public static final Relationship REL_TRANSLATION_FAILED = new Relationship.Builder() - .name("translation.failure") - .description("This relationship is used if the translation cannot be performed for some reason other than communications failure") - .build(); + .name("translation.failure") + .description("This relationship is used if the translation cannot be performed for some reason other than communications failure") + .build(); private List<PropertyDescriptor> descriptors; private Set<Relationship> relationships; - + private volatile Client client; private static final String URL = "https://translate.yandex.net/api/v1.5/tr.json/translate";; - + @Override protected void init(final ProcessorInitializationContext context) { - final List<PropertyDescriptor> descriptors = new ArrayList<PropertyDescriptor>(); + final List<PropertyDescriptor> descriptors = new ArrayList<>(); descriptors.add(KEY); descriptors.add(SOURCE_LANGUAGE); descriptors.add(TARGET_LANGUAGE); @@ -150,7 +150,7 @@ public class YandexTranslate extends AbstractProcessor { descriptors.add(CHARACTER_SET); this.descriptors = Collections.unmodifiableList(descriptors); - final Set<Relationship> relationships = new HashSet<Relationship>(); + final Set<Relationship> relationships = new HashSet<>(); relationships.add(REL_SUCCESS); relationships.add(REL_COMMS_FAILURE); relationships.add(REL_TRANSLATION_FAILED); @@ -166,168 +166,167 @@ public class YandexTranslate extends AbstractProcessor { public final List<PropertyDescriptor> getSupportedPropertyDescriptors() { return descriptors; } - + @Override protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { - return new PropertyDescriptor.Builder() - .name(propertyDescriptorName) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .expressionLanguageSupported(true) - .dynamic(true) - .build(); + return new PropertyDescriptor.Builder() + .name(propertyDescriptorName) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .expressionLanguageSupported(true) + .dynamic(true) + .build(); } @Override protected Collection<ValidationResult> customValidate(final ValidationContext validationContext) { - final List<ValidationResult> results = new ArrayList<>(); - if ( validationContext.getProperty(TRANSLATE_CONTENT).asBoolean().equals(Boolean.FALSE) ) { - boolean foundDynamic = false; - for ( final PropertyDescriptor descriptor : validationContext.getProperties().keySet() ) { - if ( descriptor.isDynamic() ) { - foundDynamic = true; - break; - } - } - - if ( !foundDynamic ) { - results.add(new ValidationResult.Builder().subject("Text to translate").input("<none>").valid(false).explanation("Must either set 'Translate Content' to true or add at least one user-defined property").build()); - } - } - - return results; + final List<ValidationResult> results = new ArrayList<>(); + if (validationContext.getProperty(TRANSLATE_CONTENT).asBoolean().equals(Boolean.FALSE)) { + boolean foundDynamic = false; + for (final PropertyDescriptor descriptor : validationContext.getProperties().keySet()) { + if (descriptor.isDynamic()) { + foundDynamic = true; + break; + } + } + + if (!foundDynamic) { + results.add(new ValidationResult.Builder().subject("Text to translate").input("<none>").valid(false) + .explanation("Must either set 'Translate Content' to true or add at least one user-defined property").build()); + } + } + + return results; } - + @OnScheduled public void onScheduled(final ProcessContext context) { - final ClientConfig config = new DefaultClientConfig(); + final ClientConfig config = new DefaultClientConfig(); config.getFeatures().put(JSONConfiguration.FEATURE_POJO_MAPPING, Boolean.TRUE); config.getClasses().add(ObjectMapperResolver.class); - client = Client.create(config); + client = Client.create(config); } @OnStopped public void destroyClient() { - if ( client != null ) { - client.destroy(); - } + if (client != null) { + client.destroy(); + } } - - + protected WebResource.Builder prepareResource(final String key, final List<String> text, final String sourceLanguage, final String destLanguage) { - WebResource webResource = client.resource(URL); - - final MultivaluedMap<String, String> paramMap = new MultivaluedMapImpl(); - paramMap.put("text", text); - paramMap.add("key", key); - paramMap.add("lang", sourceLanguage + "-" + destLanguage); - - WebResource.Builder builder = webResource - .accept(MediaType.APPLICATION_JSON) - .type(MediaType.APPLICATION_FORM_URLENCODED); - builder = builder.entity(paramMap); - - return builder; + WebResource webResource = client.resource(URL); + + final MultivaluedMap<String, String> paramMap = new MultivaluedMapImpl(); + paramMap.put("text", text); + paramMap.add("key", key); + paramMap.add("lang", sourceLanguage + "-" + destLanguage); + + WebResource.Builder builder = webResource + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_FORM_URLENCODED); + builder = builder.entity(paramMap); + + return builder; } - - + @Override public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { - FlowFile flowFile = session.get(); - if ( flowFile == null ) { - return; - } - - final StopWatch stopWatch = new StopWatch(true); - final String key = context.getProperty(KEY).getValue(); - final String sourceLanguage = context.getProperty(SOURCE_LANGUAGE).evaluateAttributeExpressions(flowFile).getValue(); - final String targetLanguage = context.getProperty(TARGET_LANGUAGE).evaluateAttributeExpressions(flowFile).getValue(); - final String encoding = context.getProperty(CHARACTER_SET).evaluateAttributeExpressions(flowFile).getValue(); - - final List<String> attributeNames = new ArrayList<>(); - final List<String> textValues = new ArrayList<>(); - for ( final PropertyDescriptor descriptor : context.getProperties().keySet() ) { - if ( descriptor.isDynamic() ) { - attributeNames.add(descriptor.getName()); // add to list so that we know the order when the translations come back. - textValues.add(context.getProperty(descriptor).evaluateAttributeExpressions(flowFile).getValue()); - } - } - - if ( context.getProperty(TRANSLATE_CONTENT).asBoolean() ) { - final byte[] buff = new byte[(int) flowFile.getSize()]; - session.read(flowFile, new InputStreamCallback() { - @Override - public void process(final InputStream in) throws IOException { - StreamUtils.fillBuffer(in, buff); - } - }); - final String content = new String(buff, Charset.forName(encoding)); - textValues.add(content); - } - - final WebResource.Builder builder = prepareResource(key, textValues, sourceLanguage, targetLanguage); - - final ClientResponse response; - try { - response = builder.post(ClientResponse.class); - } catch (final Exception e) { - getLogger().error("Failed to make request to Yandex to transate text for {} due to {}; routing to comms.failure", new Object[] {flowFile, e}); - session.transfer(flowFile, REL_COMMS_FAILURE); - return; - } - - if ( response.getStatus() != Status.OK.getStatusCode() ) { - getLogger().error("Failed to translate text using Yandex for {}; response was {}: {}; routing to {}", new Object[] { - flowFile, response.getStatus(), response.getStatusInfo().getReasonPhrase(), REL_TRANSLATION_FAILED.getName()}); - flowFile = session.putAttribute(flowFile, "yandex.translate.failure.reason", response.getStatusInfo().getReasonPhrase()); - session.transfer(flowFile, REL_TRANSLATION_FAILED); - return; - } - - final Map<String, String> newAttributes = new HashMap<>(); - final Translation translation = response.getEntity(Translation.class); - final List<String> texts = translation.getText(); - for (int i=0; i < texts.size(); i++) { - final String text = texts.get(i); - if ( i < attributeNames.size() ) { - final String attributeName = attributeNames.get(i); - newAttributes.put(attributeName, text); - } else { - flowFile = session.write(flowFile, new OutputStreamCallback() { - @Override - public void process(final OutputStream out) throws IOException { - out.write(text.getBytes(encoding)); - } - }); - - newAttributes.put("language", targetLanguage); - } - } - - if ( !newAttributes.isEmpty() ) { - flowFile = session.putAllAttributes(flowFile, newAttributes); - } - - stopWatch.stop(); - session.transfer(flowFile, REL_SUCCESS); - getLogger().info("Successfully translated {} items for {} from {} to {} in {}; routing to success", new Object[] {texts.size(), flowFile, sourceLanguage, targetLanguage, stopWatch.getDuration()}); + FlowFile flowFile = session.get(); + if (flowFile == null) { + return; + } + + final StopWatch stopWatch = new StopWatch(true); + final String key = context.getProperty(KEY).getValue(); + final String sourceLanguage = context.getProperty(SOURCE_LANGUAGE).evaluateAttributeExpressions(flowFile).getValue(); + final String targetLanguage = context.getProperty(TARGET_LANGUAGE).evaluateAttributeExpressions(flowFile).getValue(); + final String encoding = context.getProperty(CHARACTER_SET).evaluateAttributeExpressions(flowFile).getValue(); + + final List<String> attributeNames = new ArrayList<>(); + final List<String> textValues = new ArrayList<>(); + for (final PropertyDescriptor descriptor : context.getProperties().keySet()) { + if (descriptor.isDynamic()) { + attributeNames.add(descriptor.getName()); // add to list so that we know the order when the translations come back. + textValues.add(context.getProperty(descriptor).evaluateAttributeExpressions(flowFile).getValue()); + } + } + + if (context.getProperty(TRANSLATE_CONTENT).asBoolean()) { + final byte[] buff = new byte[(int) flowFile.getSize()]; + session.read(flowFile, new InputStreamCallback() { + @Override + public void process(final InputStream in) throws IOException { + StreamUtils.fillBuffer(in, buff); + } + }); + final String content = new String(buff, Charset.forName(encoding)); + textValues.add(content); + } + + final WebResource.Builder builder = prepareResource(key, textValues, sourceLanguage, targetLanguage); + + final ClientResponse response; + try { + response = builder.post(ClientResponse.class); + } catch (final Exception e) { + getLogger().error("Failed to make request to Yandex to transate text for {} due to {}; routing to comms.failure", new Object[]{flowFile, e}); + session.transfer(flowFile, REL_COMMS_FAILURE); + return; + } + + if (response.getStatus() != Status.OK.getStatusCode()) { + getLogger().error("Failed to translate text using Yandex for {}; response was {}: {}; routing to {}", new Object[]{ + flowFile, response.getStatus(), response.getStatusInfo().getReasonPhrase(), REL_TRANSLATION_FAILED.getName()}); + flowFile = session.putAttribute(flowFile, "yandex.translate.failure.reason", response.getStatusInfo().getReasonPhrase()); + session.transfer(flowFile, REL_TRANSLATION_FAILED); + return; + } + + final Map<String, String> newAttributes = new HashMap<>(); + final Translation translation = response.getEntity(Translation.class); + final List<String> texts = translation.getText(); + for (int i = 0; i < texts.size(); i++) { + final String text = texts.get(i); + if (i < attributeNames.size()) { + final String attributeName = attributeNames.get(i); + newAttributes.put(attributeName, text); + } else { + flowFile = session.write(flowFile, new OutputStreamCallback() { + @Override + public void process(final OutputStream out) throws IOException { + out.write(text.getBytes(encoding)); + } + }); + + newAttributes.put("language", targetLanguage); + } + } + + if (!newAttributes.isEmpty()) { + flowFile = session.putAllAttributes(flowFile, newAttributes); + } + + stopWatch.stop(); + session.transfer(flowFile, REL_SUCCESS); + getLogger().info("Successfully translated {} items for {} from {} to {} in {}; routing to success", + new Object[]{texts.size(), flowFile, sourceLanguage, targetLanguage, stopWatch.getDuration()}); } - private static class LanguageNameValidator implements Validator { - - @Override - public ValidationResult validate(final String subject, final String input, final ValidationContext context) { - if ( context.isExpressionLanguagePresent(input) ) { - return new ValidationResult.Builder().subject(subject).input(input).valid(true).explanation("Expression Language Present").build(); - } - - if ( Languages.getLanguageMap().keySet().contains(input.toLowerCase()) ) { - return new ValidationResult.Builder().subject(subject).input(input).valid(true).build(); - } - - return new ValidationResult.Builder().subject(subject).input(input).valid(false).explanation(input + " is not a language that is supported by Yandex").build(); - } - + + @Override + public ValidationResult validate(final String subject, final String input, final ValidationContext context) { + if (context.isExpressionLanguagePresent(input)) { + return new ValidationResult.Builder().subject(subject).input(input).valid(true).explanation("Expression Language Present").build(); + } + + if (Languages.getLanguageMap().keySet().contains(input.toLowerCase())) { + return new ValidationResult.Builder().subject(subject).input(input).valid(true).build(); + } + + return new ValidationResult.Builder().subject(subject).input(input).valid(false).explanation(input + " is not a language that is supported by Yandex").build(); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/c14b2ceb/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/main/java/org/apache/nifi/processors/yandex/model/Translation.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/main/java/org/apache/nifi/processors/yandex/model/Translation.java b/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/main/java/org/apache/nifi/processors/yandex/model/Translation.java index eeb7c3f..0e9ce84 100644 --- a/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/main/java/org/apache/nifi/processors/yandex/model/Translation.java +++ b/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/main/java/org/apache/nifi/processors/yandex/model/Translation.java @@ -22,31 +22,32 @@ import javax.xml.bind.annotation.XmlRootElement; @XmlRootElement(name = "translation") public class Translation { - private int code; - private String lang; - private List<String> text; - - public int getCode() { - return code; - } - - public void setCode(final int code) { - this.code = code; - } - - public String getLang() { - return lang; - } - - public void setLang(final String lang) { - this.lang = lang; - } - - public List<String> getText() { - return text; - } - - public void setText(final List<String> text) { - this.text = text; - } + + private int code; + private String lang; + private List<String> text; + + public int getCode() { + return code; + } + + public void setCode(final int code) { + this.code = code; + } + + public String getLang() { + return lang; + } + + public void setLang(final String lang) { + this.lang = lang; + } + + public List<String> getText() { + return text; + } + + public void setText(final List<String> text) { + this.text = text; + } } http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/c14b2ceb/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/main/java/org/apache/nifi/processors/yandex/util/Languages.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/main/java/org/apache/nifi/processors/yandex/util/Languages.java b/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/main/java/org/apache/nifi/processors/yandex/util/Languages.java index 791d6a3..754bc16 100644 --- a/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/main/java/org/apache/nifi/processors/yandex/util/Languages.java +++ b/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/main/java/org/apache/nifi/processors/yandex/util/Languages.java @@ -21,66 +21,66 @@ import java.util.HashMap; import java.util.Map; public class Languages { - private static final Map<String, String> languageAbbreviationMap = new HashMap<>(); - - static { - languageAbbreviationMap.put("ar", "arabic"); - languageAbbreviationMap.put("az", "azerbaijani"); - languageAbbreviationMap.put("be", "belarusian"); - languageAbbreviationMap.put("bg", "bulgarian"); - languageAbbreviationMap.put("bs", "bosnian"); - languageAbbreviationMap.put("ca", "catalan"); - languageAbbreviationMap.put("cs", "czech"); - languageAbbreviationMap.put("da", "danish"); - languageAbbreviationMap.put("de", "german"); - languageAbbreviationMap.put("el", "greek"); - languageAbbreviationMap.put("en", "english"); - languageAbbreviationMap.put("es", "spanish"); - languageAbbreviationMap.put("et", "estonian"); - languageAbbreviationMap.put("fi", "finnish"); - languageAbbreviationMap.put("fr", "french"); - languageAbbreviationMap.put("he", "hebrew"); - languageAbbreviationMap.put("hr", "croatian"); - languageAbbreviationMap.put("hu", "hungarian"); - languageAbbreviationMap.put("hy", "armenian"); - languageAbbreviationMap.put("id", "indonesian"); - languageAbbreviationMap.put("is", "icelandic"); - languageAbbreviationMap.put("it", "italian"); - languageAbbreviationMap.put("ja", "japanese"); - languageAbbreviationMap.put("ka", "georgian"); - languageAbbreviationMap.put("ko", "korean"); - languageAbbreviationMap.put("lt", "lithuanian"); - languageAbbreviationMap.put("lv", "latvian"); - languageAbbreviationMap.put("mk", "macedonian"); - languageAbbreviationMap.put("ms", "malay"); - languageAbbreviationMap.put("mt", "maltese"); - languageAbbreviationMap.put("nl", "dutch"); - languageAbbreviationMap.put("no", "norwegian"); - languageAbbreviationMap.put("pl", "polish"); - languageAbbreviationMap.put("pt", "portuguese"); - languageAbbreviationMap.put("ro", "romanian"); - languageAbbreviationMap.put("ru", "russian"); - languageAbbreviationMap.put("sk", "slovak"); - languageAbbreviationMap.put("sl", "slovenian"); - languageAbbreviationMap.put("sq", "albanian"); - languageAbbreviationMap.put("sr", "serbian"); - languageAbbreviationMap.put("sv", "swedish"); - languageAbbreviationMap.put("th", "thai"); - languageAbbreviationMap.put("tr", "turkish"); - languageAbbreviationMap.put("uk", "ukrainian"); - languageAbbreviationMap.put("vi", "vietnamese"); - languageAbbreviationMap.put("zh", "chinese"); - - final Map<String, String> reverseMap = new HashMap<>(); - for ( final Map.Entry<String, String> entry : languageAbbreviationMap.entrySet() ) { - reverseMap.put(entry.getValue(), entry.getKey()); - } - - languageAbbreviationMap.putAll(reverseMap); - } - - - public static Map<String, String> getLanguageMap() { - return Collections.unmodifiableMap(languageAbbreviationMap); - } + + private static final Map<String, String> languageAbbreviationMap = new HashMap<>(); + + static { + languageAbbreviationMap.put("ar", "arabic"); + languageAbbreviationMap.put("az", "azerbaijani"); + languageAbbreviationMap.put("be", "belarusian"); + languageAbbreviationMap.put("bg", "bulgarian"); + languageAbbreviationMap.put("bs", "bosnian"); + languageAbbreviationMap.put("ca", "catalan"); + languageAbbreviationMap.put("cs", "czech"); + languageAbbreviationMap.put("da", "danish"); + languageAbbreviationMap.put("de", "german"); + languageAbbreviationMap.put("el", "greek"); + languageAbbreviationMap.put("en", "english"); + languageAbbreviationMap.put("es", "spanish"); + languageAbbreviationMap.put("et", "estonian"); + languageAbbreviationMap.put("fi", "finnish"); + languageAbbreviationMap.put("fr", "french"); + languageAbbreviationMap.put("he", "hebrew"); + languageAbbreviationMap.put("hr", "croatian"); + languageAbbreviationMap.put("hu", "hungarian"); + languageAbbreviationMap.put("hy", "armenian"); + languageAbbreviationMap.put("id", "indonesian"); + languageAbbreviationMap.put("is", "icelandic"); + languageAbbreviationMap.put("it", "italian"); + languageAbbreviationMap.put("ja", "japanese"); + languageAbbreviationMap.put("ka", "georgian"); + languageAbbreviationMap.put("ko", "korean"); + languageAbbreviationMap.put("lt", "lithuanian"); + languageAbbreviationMap.put("lv", "latvian"); + languageAbbreviationMap.put("mk", "macedonian"); + languageAbbreviationMap.put("ms", "malay"); + languageAbbreviationMap.put("mt", "maltese"); + languageAbbreviationMap.put("nl", "dutch"); + languageAbbreviationMap.put("no", "norwegian"); + languageAbbreviationMap.put("pl", "polish"); + languageAbbreviationMap.put("pt", "portuguese"); + languageAbbreviationMap.put("ro", "romanian"); + languageAbbreviationMap.put("ru", "russian"); + languageAbbreviationMap.put("sk", "slovak"); + languageAbbreviationMap.put("sl", "slovenian"); + languageAbbreviationMap.put("sq", "albanian"); + languageAbbreviationMap.put("sr", "serbian"); + languageAbbreviationMap.put("sv", "swedish"); + languageAbbreviationMap.put("th", "thai"); + languageAbbreviationMap.put("tr", "turkish"); + languageAbbreviationMap.put("uk", "ukrainian"); + languageAbbreviationMap.put("vi", "vietnamese"); + languageAbbreviationMap.put("zh", "chinese"); + + final Map<String, String> reverseMap = new HashMap<>(); + for (final Map.Entry<String, String> entry : languageAbbreviationMap.entrySet()) { + reverseMap.put(entry.getValue(), entry.getKey()); + } + + languageAbbreviationMap.putAll(reverseMap); + } + + public static Map<String, String> getLanguageMap() { + return Collections.unmodifiableMap(languageAbbreviationMap); + } } http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/c14b2ceb/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/test/java/org/apache/nifi/processors/yandex/TestYandexTranslate.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/test/java/org/apache/nifi/processors/yandex/TestYandexTranslate.java b/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/test/java/org/apache/nifi/processors/yandex/TestYandexTranslate.java index 7b38b8b..8c1aaff 100644 --- a/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/test/java/org/apache/nifi/processors/yandex/TestYandexTranslate.java +++ b/nifi/nifi-nar-bundles/nifi-language-translation-bundle/nifi-yandex-processors/src/test/java/org/apache/nifi/processors/yandex/TestYandexTranslate.java @@ -42,185 +42,182 @@ import com.sun.jersey.api.client.WebResource.Builder; public class TestYandexTranslate { - private static final Map<String, String> translations = new HashMap<>(); - - @BeforeClass - public static void setupTranslationMap() { - translations.put("bonjour", "hello"); - translations.put("traduire", "translate"); - translations.put("amusant", "fun"); - translations.put("ordinateur", "computer"); - } - + private static final Map<String, String> translations = new HashMap<>(); + + @BeforeClass + public static void setupTranslationMap() { + translations.put("bonjour", "hello"); + translations.put("traduire", "translate"); + translations.put("amusant", "fun"); + translations.put("ordinateur", "computer"); + } + private TestRunner createTestRunner(final int statusCode) { - return TestRunners.newTestRunner(new YandexTranslate() { - @Override - protected Builder prepareResource(final String key, final List<String> text, final String sourceLanguage, final String destLanguage) { - final WebResource.Builder builder = Mockito.mock(WebResource.Builder.class); - - Mockito.doAnswer(new Answer<ClientResponse>() { - @Override - public ClientResponse answer(final InvocationOnMock invocation) throws Throwable { - final ClientResponse response = Mockito.mock(ClientResponse.class); - - final StatusType statusType = new StatusType() { - @Override - public int getStatusCode() { - return statusCode; - } - - @Override - public String getReasonPhrase() { - return String.valueOf(statusCode); - } - - @Override - public Family getFamily() { - return statusCode == 200 ? Family.SUCCESSFUL : Family.SERVER_ERROR; - } - }; - - Mockito.when(response.getStatus()).thenReturn(statusCode); - Mockito.when(response.getStatusInfo()).thenReturn(statusType); - - if ( statusCode == 200 ) { - final Translation translation = new Translation(); - translation.setCode(statusCode); - translation.setLang(destLanguage); - - final List<String> translationList = new ArrayList<>(); - for ( final String original : text ) { - final String translated = translations.get(original); - translationList.add(translated == null ? original : translated); - } - - translation.setText(translationList); - - Mockito.when(response.getEntity(Translation.class)).thenReturn(translation); - } - - return response; - } - }).when(builder).post(ClientResponse.class); - return builder; - } + return TestRunners.newTestRunner(new YandexTranslate() { + @Override + protected Builder prepareResource(final String key, final List<String> text, final String sourceLanguage, final String destLanguage) { + final WebResource.Builder builder = Mockito.mock(WebResource.Builder.class); + + Mockito.doAnswer(new Answer<ClientResponse>() { + @Override + public ClientResponse answer(final InvocationOnMock invocation) throws Throwable { + final ClientResponse response = Mockito.mock(ClientResponse.class); + + final StatusType statusType = new StatusType() { + @Override + public int getStatusCode() { + return statusCode; + } + + @Override + public String getReasonPhrase() { + return String.valueOf(statusCode); + } + + @Override + public Family getFamily() { + return statusCode == 200 ? Family.SUCCESSFUL : Family.SERVER_ERROR; + } + }; + + Mockito.when(response.getStatus()).thenReturn(statusCode); + Mockito.when(response.getStatusInfo()).thenReturn(statusType); + + if (statusCode == 200) { + final Translation translation = new Translation(); + translation.setCode(statusCode); + translation.setLang(destLanguage); + + final List<String> translationList = new ArrayList<>(); + for (final String original : text) { + final String translated = translations.get(original); + translationList.add(translated == null ? original : translated); + } + + translation.setText(translationList); + + Mockito.when(response.getEntity(Translation.class)).thenReturn(translation); + } + + return response; + } + }).when(builder).post(ClientResponse.class); + return builder; + } }); } - - + @Test public void testTranslateContent() { - final TestRunner testRunner = createTestRunner(200); - testRunner.setProperty(YandexTranslate.KEY, "a"); - testRunner.setProperty(YandexTranslate.SOURCE_LANGUAGE, "fr"); - testRunner.setProperty(YandexTranslate.TARGET_LANGUAGE, "en"); - testRunner.setProperty(YandexTranslate.TRANSLATE_CONTENT, "true"); - testRunner.setProperty(YandexTranslate.CHARACTER_SET, "UTF-8"); - - testRunner.enqueue("bonjour".getBytes()); - testRunner.run(); - - testRunner.assertAllFlowFilesTransferred(YandexTranslate.REL_SUCCESS, 1); - final MockFlowFile out = testRunner.getFlowFilesForRelationship(YandexTranslate.REL_SUCCESS).get(0); - - final String outText = new String(out.toByteArray()); - assertEquals("hello", outText); + final TestRunner testRunner = createTestRunner(200); + testRunner.setProperty(YandexTranslate.KEY, "a"); + testRunner.setProperty(YandexTranslate.SOURCE_LANGUAGE, "fr"); + testRunner.setProperty(YandexTranslate.TARGET_LANGUAGE, "en"); + testRunner.setProperty(YandexTranslate.TRANSLATE_CONTENT, "true"); + testRunner.setProperty(YandexTranslate.CHARACTER_SET, "UTF-8"); + + testRunner.enqueue("bonjour".getBytes()); + testRunner.run(); + + testRunner.assertAllFlowFilesTransferred(YandexTranslate.REL_SUCCESS, 1); + final MockFlowFile out = testRunner.getFlowFilesForRelationship(YandexTranslate.REL_SUCCESS).get(0); + + final String outText = new String(out.toByteArray()); + assertEquals("hello", outText); } - @Test public void testTranslateSingleAttribute() { - final TestRunner testRunner = createTestRunner(200); - - testRunner.setProperty(YandexTranslate.KEY, "A"); - testRunner.setProperty(YandexTranslate.SOURCE_LANGUAGE, "fr"); - testRunner.setProperty(YandexTranslate.TARGET_LANGUAGE, "en"); - testRunner.setProperty(YandexTranslate.TRANSLATE_CONTENT, "false"); - testRunner.setProperty(YandexTranslate.CHARACTER_SET, "UTF-8"); - testRunner.setProperty("translated", "bonjour"); - - testRunner.enqueue(new byte[0]); - testRunner.run(); - - testRunner.assertAllFlowFilesTransferred(YandexTranslate.REL_SUCCESS, 1); - final MockFlowFile out = testRunner.getFlowFilesForRelationship(YandexTranslate.REL_SUCCESS).get(0); - - assertEquals(0, out.toByteArray().length); - out.assertAttributeEquals("translated", "hello"); + final TestRunner testRunner = createTestRunner(200); + + testRunner.setProperty(YandexTranslate.KEY, "A"); + testRunner.setProperty(YandexTranslate.SOURCE_LANGUAGE, "fr"); + testRunner.setProperty(YandexTranslate.TARGET_LANGUAGE, "en"); + testRunner.setProperty(YandexTranslate.TRANSLATE_CONTENT, "false"); + testRunner.setProperty(YandexTranslate.CHARACTER_SET, "UTF-8"); + testRunner.setProperty("translated", "bonjour"); + + testRunner.enqueue(new byte[0]); + testRunner.run(); + + testRunner.assertAllFlowFilesTransferred(YandexTranslate.REL_SUCCESS, 1); + final MockFlowFile out = testRunner.getFlowFilesForRelationship(YandexTranslate.REL_SUCCESS).get(0); + + assertEquals(0, out.toByteArray().length); + out.assertAttributeEquals("translated", "hello"); } - + @Test public void testTranslateMultipleAttributes() { - final TestRunner testRunner = createTestRunner(200); - - testRunner.setProperty(YandexTranslate.KEY, "A"); - testRunner.setProperty(YandexTranslate.SOURCE_LANGUAGE, "fr"); - testRunner.setProperty(YandexTranslate.TARGET_LANGUAGE, "en"); - testRunner.setProperty(YandexTranslate.TRANSLATE_CONTENT, "false"); - testRunner.setProperty(YandexTranslate.CHARACTER_SET, "UTF-8"); - testRunner.setProperty("hello", "bonjour"); - testRunner.setProperty("translate", "traduire"); - testRunner.setProperty("fun", "amusant"); - - testRunner.enqueue(new byte[0]); - testRunner.run(); - - testRunner.assertAllFlowFilesTransferred(YandexTranslate.REL_SUCCESS, 1); - final MockFlowFile out = testRunner.getFlowFilesForRelationship(YandexTranslate.REL_SUCCESS).get(0); - - assertEquals(0, out.toByteArray().length); - out.assertAttributeEquals("hello", "hello"); - out.assertAttributeEquals("translate", "translate"); - out.assertAttributeEquals("fun", "fun"); + final TestRunner testRunner = createTestRunner(200); + + testRunner.setProperty(YandexTranslate.KEY, "A"); + testRunner.setProperty(YandexTranslate.SOURCE_LANGUAGE, "fr"); + testRunner.setProperty(YandexTranslate.TARGET_LANGUAGE, "en"); + testRunner.setProperty(YandexTranslate.TRANSLATE_CONTENT, "false"); + testRunner.setProperty(YandexTranslate.CHARACTER_SET, "UTF-8"); + testRunner.setProperty("hello", "bonjour"); + testRunner.setProperty("translate", "traduire"); + testRunner.setProperty("fun", "amusant"); + + testRunner.enqueue(new byte[0]); + testRunner.run(); + + testRunner.assertAllFlowFilesTransferred(YandexTranslate.REL_SUCCESS, 1); + final MockFlowFile out = testRunner.getFlowFilesForRelationship(YandexTranslate.REL_SUCCESS).get(0); + + assertEquals(0, out.toByteArray().length); + out.assertAttributeEquals("hello", "hello"); + out.assertAttributeEquals("translate", "translate"); + out.assertAttributeEquals("fun", "fun"); } - @Test public void testTranslateContentAndMultipleAttributes() { - final TestRunner testRunner = createTestRunner(200); - - testRunner.setProperty(YandexTranslate.KEY, "A"); - testRunner.setProperty(YandexTranslate.SOURCE_LANGUAGE, "fr"); - testRunner.setProperty(YandexTranslate.TARGET_LANGUAGE, "en"); - testRunner.setProperty(YandexTranslate.TRANSLATE_CONTENT, "true"); - testRunner.setProperty(YandexTranslate.CHARACTER_SET, "UTF-8"); - testRunner.setProperty("hello", "bonjour"); - testRunner.setProperty("translate", "traduire"); - testRunner.setProperty("fun", "amusant"); - testRunner.setProperty("nifi", "nifi"); - - testRunner.enqueue("ordinateur".getBytes()); - testRunner.run(); - - testRunner.assertAllFlowFilesTransferred(YandexTranslate.REL_SUCCESS, 1); - final MockFlowFile out = testRunner.getFlowFilesForRelationship(YandexTranslate.REL_SUCCESS).get(0); - - out.assertContentEquals("computer"); - - out.assertAttributeEquals("hello", "hello"); - out.assertAttributeEquals("translate", "translate"); - out.assertAttributeEquals("fun", "fun"); - out.assertAttributeEquals("nifi", "nifi"); + final TestRunner testRunner = createTestRunner(200); + + testRunner.setProperty(YandexTranslate.KEY, "A"); + testRunner.setProperty(YandexTranslate.SOURCE_LANGUAGE, "fr"); + testRunner.setProperty(YandexTranslate.TARGET_LANGUAGE, "en"); + testRunner.setProperty(YandexTranslate.TRANSLATE_CONTENT, "true"); + testRunner.setProperty(YandexTranslate.CHARACTER_SET, "UTF-8"); + testRunner.setProperty("hello", "bonjour"); + testRunner.setProperty("translate", "traduire"); + testRunner.setProperty("fun", "amusant"); + testRunner.setProperty("nifi", "nifi"); + + testRunner.enqueue("ordinateur".getBytes()); + testRunner.run(); + + testRunner.assertAllFlowFilesTransferred(YandexTranslate.REL_SUCCESS, 1); + final MockFlowFile out = testRunner.getFlowFilesForRelationship(YandexTranslate.REL_SUCCESS).get(0); + + out.assertContentEquals("computer"); + + out.assertAttributeEquals("hello", "hello"); + out.assertAttributeEquals("translate", "translate"); + out.assertAttributeEquals("fun", "fun"); + out.assertAttributeEquals("nifi", "nifi"); } - + @Test public void testFailureResponse() { - final TestRunner testRunner = createTestRunner(403); - - testRunner.setProperty(YandexTranslate.KEY, "A"); - testRunner.setProperty(YandexTranslate.SOURCE_LANGUAGE, "fr"); - testRunner.setProperty(YandexTranslate.TARGET_LANGUAGE, "en"); - testRunner.setProperty(YandexTranslate.TRANSLATE_CONTENT, "true"); - testRunner.setProperty(YandexTranslate.CHARACTER_SET, "UTF-8"); - testRunner.setProperty("hello", "bonjour"); - testRunner.setProperty("translate", "traduire"); - testRunner.setProperty("fun", "amusant"); - testRunner.setProperty("nifi", "nifi"); - - testRunner.enqueue("ordinateur".getBytes()); - testRunner.run(); - - testRunner.assertAllFlowFilesTransferred(YandexTranslate.REL_TRANSLATION_FAILED, 1); + final TestRunner testRunner = createTestRunner(403); + + testRunner.setProperty(YandexTranslate.KEY, "A"); + testRunner.setProperty(YandexTranslate.SOURCE_LANGUAGE, "fr"); + testRunner.setProperty(YandexTranslate.TARGET_LANGUAGE, "en"); + testRunner.setProperty(YandexTranslate.TRANSLATE_CONTENT, "true"); + testRunner.setProperty(YandexTranslate.CHARACTER_SET, "UTF-8"); + testRunner.setProperty("hello", "bonjour"); + testRunner.setProperty("translate", "traduire"); + testRunner.setProperty("fun", "amusant"); + testRunner.setProperty("nifi", "nifi"); + + testRunner.enqueue("ordinateur".getBytes()); + testRunner.run(); + + testRunner.assertAllFlowFilesTransferred(YandexTranslate.REL_TRANSLATION_FAILED, 1); } } http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/c14b2ceb/nifi/nifi-nar-bundles/nifi-language-translation-bundle/pom.xml ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-language-translation-bundle/pom.xml b/nifi/nifi-nar-bundles/nifi-language-translation-bundle/pom.xml index 43573e7..830da21 100644 --- a/nifi/nifi-nar-bundles/nifi-language-translation-bundle/pom.xml +++ b/nifi/nifi-nar-bundles/nifi-language-translation-bundle/pom.xml @@ -30,19 +30,19 @@ <module>nifi-language-translation-nar</module> </modules> - <dependencyManagement> - <dependencies> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-client</artifactId> - <version>${jersey.version}</version> - </dependency> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-json</artifactId> - <version>${jersey.version}</version> - </dependency> - </dependencies> - </dependencyManagement> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-client</artifactId> + <version>${jersey.version}</version> + </dependency> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-json</artifactId> + <version>${jersey.version}</version> + </dependency> + </dependencies> + </dependencyManagement> </project>
