This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 4ca5cf42bb [Feature Request]: add Snowflake ID Generator Transform
(#5853)
4ca5cf42bb is described below
commit 4ca5cf42bb670183db81bd3fd904a6264343010d
Author: lance <[email protected]>
AuthorDate: Tue Oct 21 19:52:11 2025 +0800
[Feature Request]: add Snowflake ID Generator Transform (#5853)
* add Snowflake ID Generator Transform
Signed-off-by: lance <[email protected]>
* Add license header to 0078-add-snowflake-id.hpl
Signed-off-by: lance <[email protected]>
* fix checkstyle AvoidStarImport
Signed-off-by: lance <[email protected]>
* add to client, remove author, add documentation page
---------
Signed-off-by: lance <[email protected]>
Co-authored-by: Hans Van Akelyen <[email protected]>
---
assemblies/plugins/pom.xml | 6 +
.../images/transforms/icons/addsnowflakeid.svg | 41 ++++
.../pages/pipeline/transforms/snowflakeid.adoc | 42 ++++
.../transforms/0078-add-snowflake-id.hpl | 132 ++++++++++++
plugins/transforms/addsnowflakeid/pom.xml | 32 +++
.../addsnowflakeid/src/assembly/assembly.xml | 51 +++++
.../transforms/addsnowflakeid/AddSnowflakeId.java | 123 +++++++++++
.../addsnowflakeid/AddSnowflakeIdData.java | 27 +++
.../addsnowflakeid/AddSnowflakeIdDialog.java | 228 +++++++++++++++++++++
.../addsnowflakeid/AddSnowflakeIdMeta.java | 82 ++++++++
.../addsnowflakeid/SnowflakeSafeIdGenerator.java | 123 +++++++++++
.../src/main/resources/addsnowflakeid.svg | 41 ++++
.../messages/messages_en_US.properties | 33 +++
.../messages/messages_zh_CN.properties | 33 +++
.../addsnowflakeid/src/main/resources/version.xml | 20 ++
.../addsnowflakeid/AddSnowflakeIdMetaTests.java | 50 +++++
.../addsnowflakeid/AddSnowflakeIdTests.java | 141 +++++++++++++
.../SnowflakeSafeIdGeneratorTests.java | 61 ++++++
plugins/transforms/pom.xml | 1 +
19 files changed, 1267 insertions(+)
diff --git a/assemblies/plugins/pom.xml b/assemblies/plugins/pom.xml
index 28793b9e3b..f1580bc4b2 100644
--- a/assemblies/plugins/pom.xml
+++ b/assemblies/plugins/pom.xml
@@ -805,6 +805,12 @@
<version>${project.version}</version>
<type>zip</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-transform-addsnowflakeid</artifactId>
+ <version>${project.version}</version>
+ <type>zip</type>
+ </dependency>
<dependency>
<groupId>org.apache.hop</groupId>
<artifactId>hop-transform-analyticquery</artifactId>
diff --git
a/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/addsnowflakeid.svg
b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/addsnowflakeid.svg
new file mode 100644
index 0000000000..8b3c0798ec
--- /dev/null
+++
b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/addsnowflakeid.svg
@@ -0,0 +1,41 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<svg xmlns='http://www.w3.org/2000/svg' width='42' height='42' viewBox='0 0 42
42'>
+ <defs>
+ <linearGradient id='g' x1='0' x2='1' y1='0' y2='1'>
+ <stop offset='0' stop-color='#6fc1ff'/>
+ <stop offset='1' stop-color='#2b9cff'/>
+ </linearGradient>
+ <filter id='f' x='-50%' y='-50%' width='200%' height='200%'>
+ <feGaussianBlur in='SourceAlpha' stdDeviation='1' result='blur'/>
+ <feOffset in='blur' dx='0' dy='1' result='off'/>
+ <feMerge>
+ <feMergeNode in='off'/>
+ <feMergeNode in='SourceGraphic'/>
+ </feMerge>
+ </filter>
+ </defs>
+ <circle cx='21' cy='21' r='20' fill='white' stroke='rgba(43,156,255,0.12)'
stroke-width='0.8'/>
+ <circle cx='21' cy='21' r='19' fill='url(#g)' opacity='0.06'/>
+ <g transform='translate(21,21)' stroke='#2b9cff' stroke-width='1.6'
stroke-linecap='round' stroke-linejoin='round' filter='url(#f)' fill='none'>
+ <g>
+ <line x1='0' y1='-11' x2='0' y2='11'/>
+ <line x1='-9.5' y1='-5.5' x2='9.5' y2='5.5'/>
+ <line x1='-9.5' y1='5.5' x2='9.5' y2='-5.5'/>
+ </g>
+ <g stroke-width='1.2'>
+ <path d='M0 -7 L-3 -9 M0 -7 L3 -9'/>
+ <path d='M0 7 L-3 9 M0 7 L3 9'/>
+ <path d='M6 3 L8 0 M6 3 L8 6'/>
+ <path d='M-6 -3 L-8 -6 M-6 -3 L-8 0'/>
+ <path d='M6 -3 L8 -6 M6 -3 L8 0'/>
+ <path d='M-6 3 L-8 6 M-6 3 L-8 0'/>
+ </g>
+ <polygon points='0,-4 3,-2 3,2 0,4 -3,2 -3,-2' fill='#eaffff'
stroke='#2b9cff' stroke-width='1'/>
+ </g>
+ <g>
+ <rect x='7' y='6' width='3.6' height='3.6' rx='0.6' fill='#2b9cff'
opacity='0.95'/>
+ <rect x='29' y='10' width='3' height='3' rx='0.5' fill='#6fc1ff'
opacity='0.95'/>
+ <rect x='9' y='28' width='2.6' height='2.6' rx='0.4' fill='#2b9cff'
opacity='0.95'/>
+ </g>
+ <text x='27' y='33.2' font-family='Inter, Arial, Helvetica, sans-serif'
font-size='7.5' font-weight='600' fill='#0b6fb6'>ID</text>
+</svg>
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/snowflakeid.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/snowflakeid.adoc
new file mode 100644
index 0000000000..e09fa7496b
--- /dev/null
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/snowflakeid.adoc
@@ -0,0 +1,42 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:documentationPath: /pipeline/transforms/
+:language: en_US
+:description: The Snowflake id transform generates a unique id for snowflake
rows
+
+= image:transforms/icons/addsnowflakeid.svg[Snowflake Bulk Loader transform
Icon, role="image-doc-icon"] Snowflake Bulk Loader
+
+[%noheader,cols="3a,1a", role="table-no-borders" ]
+|===
+|
+== Description
+
+Generate a unique identifier for a row to insert into snowflake
+
+|
+== Supported Engines
+[%noheader,cols="2,1a",frame=none, role="table-supported-engines"]
+!===
+!Hop Engine! image:check_mark.svg[Supported, 24]
+!Spark! image:question_mark.svg[Maybe Supported, 24]
+!Flink! image:question_mark.svg[Maybe Supported, 24]
+!Dataflow! image:question_mark.svg[Maybe Supported, 24]
+!===
+|===
+
+== Options
+
diff --git a/integration-tests/transforms/0078-add-snowflake-id.hpl
b/integration-tests/transforms/0078-add-snowflake-id.hpl
new file mode 100644
index 0000000000..7a64b17ab3
--- /dev/null
+++ b/integration-tests/transforms/0078-add-snowflake-id.hpl
@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0078-add-snowflake-id</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2025/10/17 14:52:28.135</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2025/10/17 14:52:28.135</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Data grid</from>
+ <to>Generate snowflake id</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Generate snowflake id</from>
+ <to>Write to log</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Data grid</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>Tom</item>
+ </line>
+ <line>
+ <item>Tim</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>name</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Generate snowflake id</name>
+ <type>SnowflakeId</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data_center_id>2</data_center_id>
+ <machine_id>31</machine_id>
+ <val_name>snowflakeId</val_name>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>256</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Write to log</name>
+ <type>WriteToLog</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <displayHeader>Y</displayHeader>
+ <fields>
+</fields>
+ <limitRows>N</limitRows>
+ <limitRowsNumber>0</limitRowsNumber>
+ <loglevel>Basic</loglevel>
+ <attributes/>
+ <GUI>
+ <xloc>320</xloc>
+ <yloc>256</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/plugins/transforms/addsnowflakeid/pom.xml
b/plugins/transforms/addsnowflakeid/pom.xml
new file mode 100644
index 0000000000..9a2cc5cf25
--- /dev/null
+++ b/plugins/transforms/addsnowflakeid/pom.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-plugins-transforms</artifactId>
+ <version>2.16.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>hop-transform-addsnowflakeid</artifactId>
+ <packaging>jar</packaging>
+ <name>Hop Plugins Transforms Add Snowflake Id</name>
+
+</project>
diff --git a/plugins/transforms/addsnowflakeid/src/assembly/assembly.xml
b/plugins/transforms/addsnowflakeid/src/assembly/assembly.xml
new file mode 100644
index 0000000000..b4aaa18c87
--- /dev/null
+++ b/plugins/transforms/addsnowflakeid/src/assembly/assembly.xml
@@ -0,0 +1,51 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.2.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.2.0
http://maven.apache.org/xsd/assembly-2.2.0.xsd";>
+ <id>hop-transform-addsnowflakeid</id>
+ <formats>
+ <format>zip</format>
+ </formats>
+ <baseDirectory>.</baseDirectory>
+ <files>
+ <file>
+ <source>${project.basedir}/src/main/resources/version.xml</source>
+
<outputDirectory>plugins/transforms/addsnowflakeid</outputDirectory>
+ <filtered>true</filtered>
+ </file>
+ </files>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}/src/main/samples</directory>
+ <outputDirectory>config/projects/samples/</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+ <dependencySets>
+ <dependencySet>
+
+ <includes>
+
<include>org.apache.hop:hop-transform-addsnowflakeid:jar</include>
+ </includes>
+
<outputDirectory>plugins/transforms/addsnowflakeid</outputDirectory>
+ </dependencySet>
+ </dependencySets>
+</assembly>
\ No newline at end of file
diff --git
a/plugins/transforms/addsnowflakeid/src/main/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeId.java
b/plugins/transforms/addsnowflakeid/src/main/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeId.java
new file mode 100644
index 0000000000..cc3c87642b
--- /dev/null
+++
b/plugins/transforms/addsnowflakeid/src/main/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeId.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.addsnowflakeid;
+
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowDataUtil;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.BaseTransform;
+import org.apache.hop.pipeline.transform.TransformMeta;
+
+/** Adds snowflake ID is a 64-bit globally unique identifier to a stream of
rows */
+public class AddSnowflakeId extends BaseTransform<AddSnowflakeIdMeta,
AddSnowflakeIdData> {
+ private static final Class<?> PKG = AddSnowflakeId.class;
+ private SnowflakeSafeIdGenerator generator;
+
+ /**
+ * This is the base transform that forms that basis for all transforms. You
can derive from this
+ * class to implement your own transforms.
+ *
+ * @param transformMeta The TransformMeta object to run.
+ * @param meta AddSnowflakeIdMeta
+ * @param data the data object to store temporary data, database
connections, caches, result sets,
+ * hashtables etc.
+ * @param copyNr The copynumber for this transform.
+ * @param pipelineMeta The PipelineMeta of which the transform transformMeta
is part of.
+ * @param pipeline The (running) pipeline to obtain information shared among
the transforms.
+ */
+ public AddSnowflakeId(
+ TransformMeta transformMeta,
+ AddSnowflakeIdMeta meta,
+ AddSnowflakeIdData data,
+ int copyNr,
+ PipelineMeta pipelineMeta,
+ Pipeline pipeline) {
+ super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline);
+ }
+
+ @Override
+ public boolean init() {
+ if (super.init()) {
+ if (isBasic()) {
+ logBasic(
+ "snowflake dataCenter: {0},machine: {1}", meta.getDataCenterId(),
meta.getMachineId());
+ }
+
+ generator = new SnowflakeSafeIdGenerator(meta.getDataCenterId(),
meta.getMachineId());
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ public boolean processRow() throws HopException {
+ // Get row from input rowSet & set row busy!
+ Object[] r = getRow();
+ if (r == null) {
+ setOutputDone();
+ return false;
+ }
+
+ if (first) {
+ first = false;
+ data.outputRowMeta = getInputRowMeta().clone();
+ meta.getFields(data.outputRowMeta, getTransformName(), null, null, this,
metadataProvider);
+ }
+
+ if (isRowLevel()) {
+ logRowlevel(
+ BaseMessages.getString(PKG, "AddSnowflakeId.Log.ReadRow")
+ + getLinesRead()
+ + " : "
+ + getInputRowMeta().getString(r));
+ }
+
+ try {
+ IRowMeta outputRowMeta = data.outputRowMeta;
+ // resize array
+ Object[] outputRowData = RowDataUtil.resizeArray(r,
outputRowMeta.size());
+ outputRowData[outputRowMeta.size() - 1] = generator.nextId();
+
+ putRow(data.outputRowMeta, outputRowData);
+
+ if (isRowLevel()) {
+ logRowlevel(
+ BaseMessages.getString(PKG, "AddSnowflakeId.Log.WriteRow")
+ + getLinesWritten()
+ + " : "
+ + getInputRowMeta().getString(r));
+ }
+
+ if (checkFeedback(getLinesRead()) && isBasic()) {
+ logBasic(BaseMessages.getString(PKG, "AddSnowflakeId.Log.LineNumber")
+ getLinesRead());
+ }
+ } catch (HopException e) {
+ logError(BaseMessages.getString(PKG,
"AddSnowflakeId.Log.ErrorInTransform") + e.getMessage());
+ setErrors(1);
+ stopAll();
+ setOutputDone();
+ return false;
+ }
+
+ return true;
+ }
+}
diff --git
a/plugins/transforms/addsnowflakeid/src/main/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeIdData.java
b/plugins/transforms/addsnowflakeid/src/main/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeIdData.java
new file mode 100644
index 0000000000..2b41b19bd9
--- /dev/null
+++
b/plugins/transforms/addsnowflakeid/src/main/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeIdData.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.addsnowflakeid;
+
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.pipeline.transform.BaseTransformData;
+import org.apache.hop.pipeline.transform.ITransformData;
+
+/** AddSnowflakeId data */
+public class AddSnowflakeIdData extends BaseTransformData implements
ITransformData {
+ public IRowMeta outputRowMeta;
+}
diff --git
a/plugins/transforms/addsnowflakeid/src/main/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeIdDialog.java
b/plugins/transforms/addsnowflakeid/src/main/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeIdDialog.java
new file mode 100644
index 0000000000..c7f1e451ce
--- /dev/null
+++
b/plugins/transforms/addsnowflakeid/src/main/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeIdDialog.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.addsnowflakeid;
+
+import org.apache.hop.core.util.Utils;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.ui.core.FormDataBuilder;
+import org.apache.hop.ui.core.PropsUi;
+import org.apache.hop.ui.core.dialog.BaseDialog;
+import org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.events.ModifyListener;
+import org.eclipse.swt.layout.FormAttachment;
+import org.eclipse.swt.layout.FormData;
+import org.eclipse.swt.layout.FormLayout;
+import org.eclipse.swt.widgets.Button;
+import org.eclipse.swt.widgets.Label;
+import org.eclipse.swt.widgets.Shell;
+import org.eclipse.swt.widgets.Spinner;
+import org.eclipse.swt.widgets.Text;
+
+/** add snowflakeId dialog */
+public class AddSnowflakeIdDialog extends BaseTransformDialog {
+ private static final Class<?> PKG = AddSnowflakeIdDialog.class;
+
+ private Text wValueName;
+ private Spinner wDataCenterId;
+ private Spinner wMachineId;
+
+ private final AddSnowflakeIdMeta input;
+
+ public AddSnowflakeIdDialog(
+ Shell parent,
+ IVariables variables,
+ AddSnowflakeIdMeta transformMeta,
+ PipelineMeta pipelineMeta) {
+ super(parent, variables, transformMeta, pipelineMeta);
+ input = transformMeta;
+ }
+
+ @Override
+ public String open() {
+ Shell parent = getParent();
+
+ shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX |
SWT.MIN);
+ PropsUi.setLook(shell);
+ setShellImage(shell, input);
+
+ ModifyListener lsMod = e -> input.setChanged();
+ changed = input.hasChanged();
+
+ FormLayout formLayout = new FormLayout();
+ formLayout.marginWidth = PropsUi.getFormMargin();
+ formLayout.marginHeight = PropsUi.getFormMargin();
+
+ shell.setLayout(formLayout);
+ shell.setText(BaseMessages.getString(PKG,
"AddSnowflakeIdDialog.Shell.Title"));
+
+ int middle = props.getMiddlePct();
+ int margin = PropsUi.getMargin();
+
+ // TransformName line
+ wlTransformName = new Label(shell, SWT.RIGHT);
+ wlTransformName.setText(
+ BaseMessages.getString(PKG,
"AddSnowflakeIdDialog.TransformName.Label"));
+ PropsUi.setLook(wlTransformName);
+ fdlTransformName = new FormData();
+ fdlTransformName.left = new FormAttachment(0, 0);
+ fdlTransformName.right = new FormAttachment(middle, -margin);
+ fdlTransformName.top = new FormAttachment(0, margin);
+ wlTransformName.setLayoutData(fdlTransformName);
+ wTransformName = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ wTransformName.setText(transformName);
+ wTransformName.addModifyListener(lsMod);
+ fdTransformName = new FormData();
+ fdTransformName.left = new FormAttachment(middle, 0);
+ fdTransformName.top = new FormAttachment(0, margin);
+ fdTransformName.right = new FormAttachment(100, 0);
+ wTransformName.setLayoutData(fdTransformName);
+ PropsUi.setLook(wTransformName);
+
+ // Value name line
+ Label wlValueName = new Label(shell, SWT.RIGHT);
+ wlValueName.setText(BaseMessages.getString(PKG,
"AddSnowflakeIdDialog.ValueName.Label"));
+ PropsUi.setLook(wlValueName);
+ wlValueName.setLayoutData(
+ new FormDataBuilder()
+ .left(0, 0)
+ .right(middle, -margin)
+ .top(wTransformName, margin)
+ .result());
+ wValueName = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ wValueName.setText("");
+ PropsUi.setLook(wValueName);
+ wValueName.addModifyListener(lsMod);
+ FormData fdValueName = new FormData();
+ fdValueName.left = new FormAttachment(middle, 5);
+ fdValueName.top = new FormAttachment(wTransformName, margin);
+ fdValueName.right = new FormAttachment(100, 0);
+ wValueName.setLayoutData(fdValueName);
+
+ // data center no
+ Label lDataCenterId = new Label(shell, SWT.RIGHT);
+ lDataCenterId.setText(BaseMessages.getString(PKG,
"AddSnowflakeIdDialog.DataCenterNo.Label"));
+ lDataCenterId.setLayoutData(
+ new FormDataBuilder().left(0, 0).right(middle,
-margin).top(wlValueName, margin).result());
+ PropsUi.setLook(lDataCenterId);
+ wDataCenterId = createSpinner();
+ PropsUi.setLook(wDataCenterId);
+ wDataCenterId.addModifyListener(lsMod);
+ wDataCenterId.setLayoutData(
+ new FormDataBuilder().left(middle, 5).right(100, 0).top(wValueName,
margin).result());
+
+ // machine no
+ Label lMachine = new Label(shell, SWT.RIGHT);
+ lMachine.setText(BaseMessages.getString(PKG,
"AddSnowflakeIdDialog.MachineNo.Label"));
+ PropsUi.setLook(lMachine);
+ lMachine.setLayoutData(
+ new FormDataBuilder()
+ .left(0, 0)
+ .right(middle, -margin)
+ .top(wDataCenterId, margin)
+ .result());
+
+ wMachineId = createSpinner();
+ PropsUi.setLook(wMachineId);
+ wMachineId.addModifyListener(lsMod);
+ wMachineId.setLayoutData(
+ new FormDataBuilder().left(middle, 5).right(100, 0).top(wDataCenterId,
margin).result());
+
+ // THE BUTTONS
+ wOk = new Button(shell, SWT.PUSH);
+ wOk.setText(BaseMessages.getString(PKG, "System.Button.OK"));
+ wCancel = new Button(shell, SWT.PUSH);
+ wCancel.setText(BaseMessages.getString(PKG, "System.Button.Cancel"));
+
+ setButtonPositions(new Button[] {wOk, wCancel}, margin, wMachineId);
+
+ // Add listeners
+ wOk.addListener(SWT.Selection, e -> ok());
+ wCancel.addListener(SWT.Selection, e -> cancel());
+
+ getData();
+
+ BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel());
+ return transformName;
+ }
+
+ /** Copy information from the meta-data input to the dialog fields. */
+ public void getData() {
+ logDebug(BaseMessages.getString(PKG,
"AddSnowflakeIdDialog.Log.GettingKeyInfo"));
+
+ if (input.getValueName() != null) {
+ wValueName.setText(input.getValueName());
+ }
+
+ wDataCenterId.setSelection(ensureRange0To32(input.getDataCenterId()));
+ wMachineId.setSelection(ensureRange0To32(input.getMachineId()));
+
+ wTransformName.selectAll();
+ wTransformName.setFocus();
+ }
+
+ private void cancel() {
+ transformName = null;
+ input.setChanged(changed);
+ dispose();
+ }
+
+ private void ok() {
+ if (Utils.isEmpty(wTransformName.getText())) {
+ return;
+ }
+
+ transformName = wTransformName.getText();
+ input.setDataCenterId(wDataCenterId.getSelection());
+ input.setMachineId(wMachineId.getSelection());
+ dispose();
+ }
+
+ /**
+ * handler value [0,32)
+ *
+ * @param value init value
+ * @return real value
+ */
+ private Integer ensureRange0To32(Integer value) {
+ if (value == null || value < 0) {
+ return 0;
+ }
+
+ if (value > 31) {
+ return 31;
+ }
+
+ return value;
+ }
+
+ /**
+ * create spinner control
+ *
+ * @return Spinner
+ */
+ private Spinner createSpinner() {
+ Spinner spinner = new Spinner(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ spinner.setMinimum(0);
+ spinner.setMaximum(31);
+ spinner.setIncrement(1);
+ return spinner;
+ }
+}
diff --git
a/plugins/transforms/addsnowflakeid/src/main/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeIdMeta.java
b/plugins/transforms/addsnowflakeid/src/main/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeIdMeta.java
new file mode 100644
index 0000000000..b09790d6b3
--- /dev/null
+++
b/plugins/transforms/addsnowflakeid/src/main/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeIdMeta.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.addsnowflakeid;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.transform.BaseTransformMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+
+/** Meta data for the Add Snowflake Id transform */
+@Transform(
+ id = "SnowflakeId",
+ image = "addsnowflakeid.svg",
+ name = "i18n::BaseTransform.TypeLongDesc.AddSnowflakeId",
+ description = "i18n::BaseTransform.TypeTooltipDesc.AddSnowflakeId",
+ categoryDescription =
"i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Transform",
+ documentationUrl = "/pipeline/transforms/snowflakeid.html",
+ keywords = "i18n::AddSnowflakeIdMeta.keyword")
+@Getter
+@Setter
+public class AddSnowflakeIdMeta extends BaseTransformMeta<AddSnowflakeId,
AddSnowflakeIdData> {
+ private static final Class<?> PKG = AddSnowflakeIdMeta.class;
+
+ @HopMetadataProperty(
+ key = "val_name",
+ injectionKeyDescription = "AddSnowflakeIdMeta.Injection.valueName")
+ private String valueName;
+
+ @HopMetadataProperty(
+ key = "data_center_id",
+ injectionKeyDescription = "AddSnowflakeIdMeta.Injection.DataCenterId")
+ private Integer dataCenterId;
+
+ @HopMetadataProperty(
+ key = "machine_id",
+ injectionKeyDescription = "AddSnowflakeIdMeta.Injection.MachineId")
+ private Integer machineId;
+
+ @Override
+ public void setDefault() {
+ valueName = "snowflakeId";
+ dataCenterId = 1;
+ machineId = 1;
+ }
+
+ @Override
+ public void getFields(
+ IRowMeta row,
+ String name,
+ IRowMeta[] info,
+ TransformMeta nextTransform,
+ IVariables variables,
+ IHopMetadataProvider provider)
+ throws HopTransformException {
+ IValueMeta v = new ValueMetaInteger(valueName);
+ v.setOrigin(name);
+ row.addValueMeta(v);
+ }
+}
diff --git
a/plugins/transforms/addsnowflakeid/src/main/java/org/apache/hop/pipeline/transforms/addsnowflakeid/SnowflakeSafeIdGenerator.java
b/plugins/transforms/addsnowflakeid/src/main/java/org/apache/hop/pipeline/transforms/addsnowflakeid/SnowflakeSafeIdGenerator.java
new file mode 100644
index 0000000000..41634e22de
--- /dev/null
+++
b/plugins/transforms/addsnowflakeid/src/main/java/org/apache/hop/pipeline/transforms/addsnowflakeid/SnowflakeSafeIdGenerator.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.addsnowflakeid;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+/** snowflake id utils */
+public class SnowflakeSafeIdGenerator {
+ /** 2020-01-01 */
+ private static final long START_EPOCH = 1577836800000L;
+
+ private static final long DATA_CENTER_ID_BITS = 5L;
+ private static final long MACHINE_ID_BITS = 5L;
+ private static final long SEQUENCE_BITS = 12L;
+
+ private static final long MAX_DATA_CENTER_ID = ~(-1L << DATA_CENTER_ID_BITS);
+ private static final long MAX_MACHINE_ID = ~(-1L << MACHINE_ID_BITS);
+ private static final long MAX_SEQUENCE = ~(-1L << SEQUENCE_BITS);
+
+ private static final long MACHINE_ID_SHIFT = SEQUENCE_BITS;
+ private static final long DATA_CENTER_ID_SHIFT = SEQUENCE_BITS +
MACHINE_ID_BITS;
+ private static final long TIMESTAMP_SHIFT = SEQUENCE_BITS + MACHINE_ID_BITS
+ DATA_CENTER_ID_BITS;
+
+ private final long dataCenterId;
+ private final long machineId;
+
+ /** Maximum tolerable rollback time in milliseconds */
+ private final long maxBackwardsMs;
+
+ private long sequence = 0L;
+ private long lastTimestamp = -1L;
+
+ /** logical timestamp */
+ private long logicalTimestamp = -1L;
+
+ public SnowflakeSafeIdGenerator(long dataCenterId, long machineId) {
+ this(dataCenterId, machineId, 10);
+ }
+
+ public SnowflakeSafeIdGenerator(long dataCenterId, long machineId, long
maxBackwardsMs) {
+ if (dataCenterId > MAX_DATA_CENTER_ID || dataCenterId < 0) {
+ throw new IllegalArgumentException("dataCenterId out of range");
+ }
+ if (machineId > MAX_MACHINE_ID || machineId < 0) {
+ throw new IllegalArgumentException("machineId out of range");
+ }
+
+ this.dataCenterId = dataCenterId;
+ this.machineId = machineId;
+ this.maxBackwardsMs = maxBackwardsMs;
+ }
+
+ /**
+ * default SnowflakeSafeIdGenerator
+ *
+ * @return SnowflakeSafeIdGenerator
+ */
+ public static SnowflakeSafeIdGenerator createDefault() {
+ long dataCenterId = ThreadLocalRandom.current().nextInt(0, 32);
+ long machineId = ThreadLocalRandom.current().nextInt(0, 32);
+ return new SnowflakeSafeIdGenerator(dataCenterId, machineId, 10);
+ }
+
+ public synchronized long nextId() {
+ long now = currentTimeMillis();
+
+ // Timestamp rollback detection
+ if (now < lastTimestamp) {
+ long offset = lastTimestamp - now;
+ if (offset <= maxBackwardsMs) {
+ // For minor clock rollbacks, continue running using logical time.
+ now = lastTimestamp;
+ } else {
+ throw new IllegalArgumentException(
+ "Clock moved backwards beyond tolerance. Refusing for " + offset +
"ms");
+ }
+ }
+
+ if (now == lastTimestamp) {
+ sequence = (sequence + 1) & MAX_SEQUENCE;
+ if (sequence == 0) {
+ now = waitUntilNextMillis(lastTimestamp);
+ }
+ } else {
+ sequence = 0;
+ }
+
+ lastTimestamp = now;
+ logicalTimestamp = Math.max(now, logicalTimestamp + 1);
+
+ return ((logicalTimestamp - START_EPOCH) << TIMESTAMP_SHIFT)
+ | (dataCenterId << DATA_CENTER_ID_SHIFT)
+ | (machineId << MACHINE_ID_SHIFT)
+ | sequence;
+ }
+
+ private long waitUntilNextMillis(long lastTime) {
+ long ts = currentTimeMillis();
+ while (ts <= lastTime) {
+ ts = currentTimeMillis();
+ }
+ return ts;
+ }
+
+ private long currentTimeMillis() {
+ return System.currentTimeMillis();
+ }
+}
diff --git
a/plugins/transforms/addsnowflakeid/src/main/resources/addsnowflakeid.svg
b/plugins/transforms/addsnowflakeid/src/main/resources/addsnowflakeid.svg
new file mode 100644
index 0000000000..8b3c0798ec
--- /dev/null
+++ b/plugins/transforms/addsnowflakeid/src/main/resources/addsnowflakeid.svg
@@ -0,0 +1,41 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<svg xmlns='http://www.w3.org/2000/svg' width='42' height='42' viewBox='0 0 42
42'>
+ <defs>
+ <linearGradient id='g' x1='0' x2='1' y1='0' y2='1'>
+ <stop offset='0' stop-color='#6fc1ff'/>
+ <stop offset='1' stop-color='#2b9cff'/>
+ </linearGradient>
+ <filter id='f' x='-50%' y='-50%' width='200%' height='200%'>
+ <feGaussianBlur in='SourceAlpha' stdDeviation='1' result='blur'/>
+ <feOffset in='blur' dx='0' dy='1' result='off'/>
+ <feMerge>
+ <feMergeNode in='off'/>
+ <feMergeNode in='SourceGraphic'/>
+ </feMerge>
+ </filter>
+ </defs>
+ <circle cx='21' cy='21' r='20' fill='white' stroke='rgba(43,156,255,0.12)'
stroke-width='0.8'/>
+ <circle cx='21' cy='21' r='19' fill='url(#g)' opacity='0.06'/>
+ <g transform='translate(21,21)' stroke='#2b9cff' stroke-width='1.6'
stroke-linecap='round' stroke-linejoin='round' filter='url(#f)' fill='none'>
+ <g>
+ <line x1='0' y1='-11' x2='0' y2='11'/>
+ <line x1='-9.5' y1='-5.5' x2='9.5' y2='5.5'/>
+ <line x1='-9.5' y1='5.5' x2='9.5' y2='-5.5'/>
+ </g>
+ <g stroke-width='1.2'>
+ <path d='M0 -7 L-3 -9 M0 -7 L3 -9'/>
+ <path d='M0 7 L-3 9 M0 7 L3 9'/>
+ <path d='M6 3 L8 0 M6 3 L8 6'/>
+ <path d='M-6 -3 L-8 -6 M-6 -3 L-8 0'/>
+ <path d='M6 -3 L8 -6 M6 -3 L8 0'/>
+ <path d='M-6 3 L-8 6 M-6 3 L-8 0'/>
+ </g>
+ <polygon points='0,-4 3,-2 3,2 0,4 -3,2 -3,-2' fill='#eaffff'
stroke='#2b9cff' stroke-width='1'/>
+ </g>
+ <g>
+ <rect x='7' y='6' width='3.6' height='3.6' rx='0.6' fill='#2b9cff'
opacity='0.95'/>
+ <rect x='29' y='10' width='3' height='3' rx='0.5' fill='#6fc1ff'
opacity='0.95'/>
+ <rect x='9' y='28' width='2.6' height='2.6' rx='0.4' fill='#2b9cff'
opacity='0.95'/>
+ </g>
+ <text x='27' y='33.2' font-family='Inter, Arial, Helvetica, sans-serif'
font-size='7.5' font-weight='600' fill='#0b6fb6'>ID</text>
+</svg>
diff --git
a/plugins/transforms/addsnowflakeid/src/main/resources/org/apache/hop/pipeline/transforms/addsnowflakeid/messages/messages_en_US.properties
b/plugins/transforms/addsnowflakeid/src/main/resources/org/apache/hop/pipeline/transforms/addsnowflakeid/messages/messages_en_US.properties
new file mode 100644
index 0000000000..b37bc3e505
--- /dev/null
+++
b/plugins/transforms/addsnowflakeid/src/main/resources/org/apache/hop/pipeline/transforms/addsnowflakeid/messages/messages_en_US.properties
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+AddSnowflakeId.Log.ErrorInTransform=Because of an error, this transform can''t
continue\:
+AddSnowflakeId.Log.LineNumber=linenr
+AddSnowflakeId.Log.ReadRow=Read row \#
+AddSnowflakeId.Log.WriteRow=Wrote row \#
+AddSnowflakeIdDialog.MachineNo.Label=Machine No
+AddSnowflakeIdDialog.Log.GettingKeyInfo=getting key info...
+AddSnowflakeIdDialog.Shell.Title=Generate snowflake id
+AddSnowflakeIdDialog.DataCenterNo.Label=Data Center No
+AddSnowflakeIdDialog.TransformName.Label=Transform name
+AddSnowflakeIdDialog.ValueName.Label=Name of value
+AddSnowflakeIdMeta.Injection.valueName=Value name
+AddSnowflakeIdMeta.Injection.DataCenterId=Data Center Id
+AddSnowflakeIdMeta.Injection.MachineId=Machine id
+AddSnowflakeIdMeta.keyword=add,generate,snowflake,id
+BaseTransform.TypeLongDesc.AddSnowflakeId=Generate snowflake id
+BaseTransform.TypeTooltipDesc.AddSnowflakeId=Get the next value from an
snowflakeId
diff --git
a/plugins/transforms/addsnowflakeid/src/main/resources/org/apache/hop/pipeline/transforms/addsnowflakeid/messages/messages_zh_CN.properties
b/plugins/transforms/addsnowflakeid/src/main/resources/org/apache/hop/pipeline/transforms/addsnowflakeid/messages/messages_zh_CN.properties
new file mode 100644
index 0000000000..52631a19ff
--- /dev/null
+++
b/plugins/transforms/addsnowflakeid/src/main/resources/org/apache/hop/pipeline/transforms/addsnowflakeid/messages/messages_zh_CN.properties
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+AddSnowflakeId.Log.ErrorInTransform=Because of an error, this transform can''t
continue\:
+AddSnowflakeId.Log.LineNumber=\u884C\u53F7
+AddSnowflakeId.Log.ReadRow=\u8BFB\u4E00\u884C \#
+AddSnowflakeId.Log.WriteRow=\u5199\u4E00\u884C \#
+AddSnowflakeIdDialog.MachineNo.Label=\u5E94\u7528\u7F16\u53F7
+AddSnowflakeIdDialog.Log.GettingKeyInfo=Getting key info...
+AddSnowflakeIdDialog.Shell.Title=\u4EA7\u751F\u96EA\u82B1\u7B97\u6CD5ID
+AddSnowflakeIdDialog.DataCenterNo.Label=\u5E94\u7528\u7F16\u53F7
+AddSnowflakeIdDialog.TransformName.Label=\u8F6C\u6362\u540D\u79F0
+AddSnowflakeIdDialog.ValueName.Label=\u751F\u6210id\u5B57\u6BB5
+AddSnowflakeIdMeta.Injection.valueName=Value name
+AddSnowflakeIdMeta.Injection.DataCenterId=Data Center Id
+AddSnowflakeIdMeta.Injection.MachineId=Machine id
+AddSnowflakeIdMeta.keyword=\u6DFB\u52A0,\u96EA\u82B1ID,ID,\u5168\u5C40\u552F\u4E00ID
+BaseTransform.TypeLongDesc.AddSnowflakeId=\u4EA7\u751F\u96EA\u82B1\u7B97\u6CD5ID
+BaseTransform.TypeTooltipDesc.AddSnowflakeId=\u83B7\u53D6\u4E0B\u4E00\u4E2A\u96EA\u82B1ID
diff --git a/plugins/transforms/addsnowflakeid/src/main/resources/version.xml
b/plugins/transforms/addsnowflakeid/src/main/resources/version.xml
new file mode 100644
index 0000000000..6be576acae
--- /dev/null
+++ b/plugins/transforms/addsnowflakeid/src/main/resources/version.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<version>${project.version}</version>
\ No newline at end of file
diff --git
a/plugins/transforms/addsnowflakeid/src/test/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeIdMetaTests.java
b/plugins/transforms/addsnowflakeid/src/test/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeIdMetaTests.java
new file mode 100644
index 0000000000..bd7aa15247
--- /dev/null
+++
b/plugins/transforms/addsnowflakeid/src/test/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeIdMetaTests.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.addsnowflakeid;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/** AddSnowflakeIdMeta test */
+class AddSnowflakeIdMetaTests {
+
+ @Test
+ void setDefault() {
+ AddSnowflakeIdMeta meta = new AddSnowflakeIdMeta();
+ meta.setDefault();
+
+ Assertions.assertEquals(1, meta.getMachineId());
+ Assertions.assertEquals(1, meta.getDataCenterId());
+ Assertions.assertEquals("snowflakeId", meta.getValueName());
+ }
+
+ @Test
+ void testClone() {
+ AddSnowflakeIdMeta meta = new AddSnowflakeIdMeta();
+ meta.setValueName("snowflake_id");
+ meta.setDataCenterId(10);
+ meta.setMachineId(20);
+
+ AddSnowflakeIdMeta cloned = (AddSnowflakeIdMeta) meta.clone();
+
+ Assertions.assertNotNull(cloned);
+ Assertions.assertNotSame(meta, cloned);
+ Assertions.assertEquals(meta.getValueName(), cloned.getValueName());
+ Assertions.assertEquals(meta.getDataCenterId(), cloned.getDataCenterId());
+ Assertions.assertEquals(meta.getMachineId(), cloned.getMachineId());
+ }
+}
diff --git
a/plugins/transforms/addsnowflakeid/src/test/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeIdTests.java
b/plugins/transforms/addsnowflakeid/src/test/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeIdTests.java
new file mode 100644
index 0000000000..cd48ef5362
--- /dev/null
+++
b/plugins/transforms/addsnowflakeid/src/test/java/org/apache/hop/pipeline/transforms/addsnowflakeid/AddSnowflakeIdTests.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.pipeline.transforms.addsnowflakeid;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.encryption.HopTwoWayPasswordEncoder;
+import org.apache.hop.core.encryption.TwoWayPasswordEncoderPluginType;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.plugins.Plugin;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.pipeline.PipelineTestingUtil;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/** AddSnowflakeId test */
+class AddSnowflakeIdTests {
+ private TransformMockHelper<AddSnowflakeIdMeta, AddSnowflakeIdData>
transformMockHelper;
+
+ @BeforeAll
+ static void setUpBeforeClass() throws HopException {
+ Map<Class<?>, String> map = new HashMap<>();
+ map.put(HopTwoWayPasswordEncoder.class,
HopTwoWayPasswordEncoder.class.getName());
+ List<String> libraries = new ArrayList<>();
+
+ Plugin plugin =
+ new Plugin(
+ new String[] {"Hop"},
+ TwoWayPasswordEncoderPluginType.class,
+ HopTwoWayPasswordEncoder.class,
+ "Encryption",
+ "Hop",
+ null,
+ null,
+ false,
+ false,
+ map,
+ libraries,
+ null,
+ null,
+ null,
+ false);
+
+
PluginRegistry.getInstance().registerPlugin(TwoWayPasswordEncoderPluginType.class,
plugin);
+
+ HopEnvironment.init();
+ }
+
+ @BeforeEach
+ void setUp() {
+ transformMockHelper =
+ new TransformMockHelper<>(
+ "AddSnowflakeId", AddSnowflakeIdMeta.class,
AddSnowflakeIdData.class);
+ when(transformMockHelper.logChannelFactory.create(any(),
any(ILoggingObject.class)))
+ .thenReturn(transformMockHelper.iLogChannel);
+ when(transformMockHelper.pipeline.isRunning()).thenReturn(true);
+
when(transformMockHelper.transformMeta.getName()).thenReturn("AddSnowflakeId");
+ }
+
+ @Test
+ void processRow() throws Exception {
+ when(transformMockHelper.iTransformMeta.getValueName()).thenReturn("id");
+ when(transformMockHelper.iTransformMeta.getDataCenterId()).thenReturn(10);
+ when(transformMockHelper.iTransformMeta.getMachineId()).thenReturn(20);
+ when(transformMockHelper.pipeline.getContainerId())
+ .thenReturn("cId-" + System.currentTimeMillis());
+
+ AddSnowflakeId addSnowflakeId =
+ new AddSnowflakeId(
+ transformMockHelper.transformMeta,
+ transformMockHelper.iTransformMeta,
+ transformMockHelper.iTransformData,
+ 0,
+ transformMockHelper.pipelineMeta,
+ transformMockHelper.pipeline);
+
+ // init success.
+ boolean result = addSnowflakeId.init();
+ assertTrue(result);
+
+ // Set up input row meta
+ RowMeta inputRowMeta = new RowMeta();
+ inputRowMeta.addValueMeta(new ValueMetaInteger("No"));
+ inputRowMeta.addValueMeta(new ValueMetaInteger("Id"));
+ addSnowflakeId.setInputRowMeta(inputRowMeta);
+
+ // spy
+ addSnowflakeId = spy(addSnowflakeId);
+ doReturn(new Object[] {1000L})
+ .doReturn(new Object[] {2000L})
+ .doReturn(new Object[] {3000L})
+ .doReturn(null)
+ .when(addSnowflakeId)
+ .getRow();
+
+ List<Object[]> execCount = PipelineTestingUtil.execute(addSnowflakeId, 3,
false);
+ assertEquals(3, execCount.size());
+ assertTrue((long) execCount.get(1)[1] > (long) execCount.get(0)[1]);
+
+ IRowMeta outputRowMeta = transformMockHelper.iTransformData.outputRowMeta;
+ assertEquals(2, outputRowMeta.size());
+
+ addSnowflakeId.dispose();
+ }
+
+ @AfterEach
+ void tearDown() {
+ transformMockHelper.cleanUp();
+ }
+}
diff --git
a/plugins/transforms/addsnowflakeid/src/test/java/org/apache/hop/pipeline/transforms/addsnowflakeid/SnowflakeSafeIdGeneratorTests.java
b/plugins/transforms/addsnowflakeid/src/test/java/org/apache/hop/pipeline/transforms/addsnowflakeid/SnowflakeSafeIdGeneratorTests.java
new file mode 100644
index 0000000000..d58f521836
--- /dev/null
+++
b/plugins/transforms/addsnowflakeid/src/test/java/org/apache/hop/pipeline/transforms/addsnowflakeid/SnowflakeSafeIdGeneratorTests.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.addsnowflakeid;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/** SnowflakeIdUtils test */
+class SnowflakeSafeIdGeneratorTests {
+
+ @Test
+ void testDefaultNextId() {
+ SnowflakeSafeIdGenerator generator =
SnowflakeSafeIdGenerator.createDefault();
+ long preId = 766649250213851136L;
+ long id = generator.nextId();
+
+ Assertions.assertTrue(id > preId);
+ }
+
+ @Test
+ void testNextId() {
+ long dataCenterId = 1;
+ long machineId = 10;
+ SnowflakeSafeIdGenerator generator = new
SnowflakeSafeIdGenerator(dataCenterId, machineId);
+
+ long preId = 766649250213851136L;
+ long id = generator.nextId();
+
+ Assertions.assertTrue(id > preId);
+ }
+
+ @Test
+ void testNextIdWithMaxBackwardsMs() {
+ long dataCenterId = 1;
+ long machineId = 10;
+ long maxBackwardsMs = 30;
+
+ SnowflakeSafeIdGenerator generator =
+ new SnowflakeSafeIdGenerator(dataCenterId, machineId, maxBackwardsMs);
+
+ long preId = 766649250213851136L;
+ long id = generator.nextId();
+
+ Assertions.assertTrue(id > preId);
+ }
+}
diff --git a/plugins/transforms/pom.xml b/plugins/transforms/pom.xml
index 29ee10336a..82d1f1c2fd 100644
--- a/plugins/transforms/pom.xml
+++ b/plugins/transforms/pom.xml
@@ -31,6 +31,7 @@
<modules>
<module>abort</module>
<module>addsequence</module>
+ <module>addsnowflakeid</module>
<module>analyticquery</module>
<module>append</module>
<module>blockingtransform</module>
