This is an automated email from the ASF dual-hosted git repository.
zhangliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 1f7f820 Fix the separator problem under window. (#12161)
1f7f820 is described below
commit 1f7f8205e2bdc59b0067185662104ca2b33c1f0a
Author: Guocheng Tang <[email protected]>
AuthorDate: Thu Sep 2 22:47:56 2021 +0800
Fix the separator problem under window. (#12161)
* Fixed #12157.
* Get system separator for yaml conversion.
* Fixes test case.
* Load config form YAML.
* Add new line.
* Load config from YAML.
* Fix code style.
---
...urationsForYamlShadowRuleConfigurationTest.java | 2 +-
...ationsForYamlShardingRuleConfigurationTest.java | 2 +-
.../infra/yaml/engine/YamlEngine.java | 6 ++--
.../swapper/YamlEngineUserConfigurationTest.java | 2 +-
.../infra/yaml/engine/YamlEngineTest.java | 11 +++++---
.../service/impl/DataSourcePersistServiceTest.java | 3 +-
.../api/impl/GovernanceRepositoryAPIImplTest.java | 17 ++----------
.../scaling/core/job/progress/JobProgressTest.java | 27 ++++--------------
.../scaling/core/util/ResourceUtil.java | 21 ++++++++++++++
.../src/test/resources/governance-repository.yaml | 29 ++++++++++++++++++++
.../src/test/resources/job-progress.yaml | 32 ++++++++++++++++++++++
11 files changed, 104 insertions(+), 48 deletions(-)
diff --git
a/shardingsphere-features/shardingsphere-shadow/shardingsphere-shadow-core/src/test/java/org/apache/shardingsphere/shadow/swapper/YamlRootRuleConfigurationsForYamlShadowRuleConfigurationTest.java
b/shardingsphere-features/shardingsphere-shadow/shardingsphere-shadow-core/src/test/java/org/apache/shardingsphere/shadow/swapper/YamlRootRuleConfigurationsForYamlShadowRuleConfigurationTest.java
index ff4f5fa..ed705e3 100644
---
a/shardingsphere-features/shardingsphere-shadow/shardingsphere-shadow-core/src/test/java/org/apache/shardingsphere/shadow/swapper/YamlRootRuleConfigurationsForYamlShadowRuleConfigurationTest.java
+++
b/shardingsphere-features/shardingsphere-shadow/shardingsphere-shadow-core/src/test/java/org/apache/shardingsphere/shadow/swapper/YamlRootRuleConfigurationsForYamlShadowRuleConfigurationTest.java
@@ -54,7 +54,7 @@ public final class
YamlRootRuleConfigurationsForYamlShadowRuleConfigurationTest
BufferedReader reader = new BufferedReader(fileReader)) {
String line;
while (null != (line = reader.readLine())) {
- yamlContent.append(line).append("\n");
+ yamlContent.append(line).append(System.lineSeparator());
}
}
YamlRootConfiguration rootConfig =
YamlEngine.unmarshal(yamlContent.toString().getBytes(),
YamlRootConfiguration.class);
diff --git
a/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/test/java/org/apache/shardingsphere/sharding/swapper/YamlRootRuleConfigurationsForYamlShardingRuleConfigurationTest.java
b/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/test/java/org/apache/shardingsphere/sharding/swapper/YamlRootRuleConfigurationsForYamlShardingRuleConfigurationTest.java
index 9ec99e6..bf32948 100644
---
a/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/test/java/org/apache/shardingsphere/sharding/swapper/YamlRootRuleConfigurationsForYamlShardingRuleConfigurationTest.java
+++
b/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/test/java/org/apache/shardingsphere/sharding/swapper/YamlRootRuleConfigurationsForYamlShardingRuleConfigurationTest.java
@@ -54,7 +54,7 @@ public final class
YamlRootRuleConfigurationsForYamlShardingRuleConfigurationTes
BufferedReader reader = new BufferedReader(fileReader)) {
String line;
while (null != (line = reader.readLine())) {
- yamlContent.append(line).append("\n");
+ yamlContent.append(line).append(System.lineSeparator());
}
}
assertYamlShardingConfiguration(YamlEngine.unmarshal(yamlContent.toString().getBytes(),
YamlRootConfiguration.class));
diff --git
a/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/yaml/engine/YamlEngine.java
b/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/yaml/engine/YamlEngine.java
index 5523039..0624839 100644
---
a/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/yaml/engine/YamlEngine.java
+++
b/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/yaml/engine/YamlEngine.java
@@ -91,9 +91,11 @@ public final class YamlEngine {
* @return YAML content
*/
public static String marshal(final Object value) {
+ DumperOptions dumperOptions = new DumperOptions();
+
dumperOptions.setLineBreak(DumperOptions.LineBreak.getPlatformLineBreak());
if (value instanceof Collection) {
- return new Yaml(new ShardingSphereYamlRepresenter()).dumpAs(value,
null, DumperOptions.FlowStyle.BLOCK);
+ return new Yaml(new ShardingSphereYamlRepresenter(),
dumperOptions).dumpAs(value, null, DumperOptions.FlowStyle.BLOCK);
}
- return new Yaml(new ShardingSphereYamlRepresenter()).dumpAsMap(value);
+ return new Yaml(new ShardingSphereYamlRepresenter(),
dumperOptions).dumpAsMap(value);
}
}
diff --git
a/shardingsphere-infra/shardingsphere-infra-common/src/test/java/org/apache/shardingsphere/infra/metadata/user/yaml/swapper/YamlEngineUserConfigurationTest.java
b/shardingsphere-infra/shardingsphere-infra-common/src/test/java/org/apache/shardingsphere/infra/metadata/user/yaml/swapper/YamlEngineUserConfigurationTest.java
index 01ce373..750be47 100644
---
a/shardingsphere-infra/shardingsphere-infra-common/src/test/java/org/apache/shardingsphere/infra/metadata/user/yaml/swapper/YamlEngineUserConfigurationTest.java
+++
b/shardingsphere-infra/shardingsphere-infra-common/src/test/java/org/apache/shardingsphere/infra/metadata/user/yaml/swapper/YamlEngineUserConfigurationTest.java
@@ -44,6 +44,6 @@ public final class YamlEngineUserConfigurationTest {
public void assertMarshal() {
YamlUserConfiguration actual = new YamlUserConfiguration();
actual.setPassword("pwd");
- assertThat(YamlEngine.marshal(actual), is("password: pwd\n"));
+ assertThat(YamlEngine.marshal(actual), is("password: pwd" +
System.lineSeparator()));
}
}
diff --git
a/shardingsphere-infra/shardingsphere-infra-common/src/test/java/org/apache/shardingsphere/infra/yaml/engine/YamlEngineTest.java
b/shardingsphere-infra/shardingsphere-infra-common/src/test/java/org/apache/shardingsphere/infra/yaml/engine/YamlEngineTest.java
index 27acbca..606a0f8 100644
---
a/shardingsphere-infra/shardingsphere-infra-common/src/test/java/org/apache/shardingsphere/infra/yaml/engine/YamlEngineTest.java
+++
b/shardingsphere-infra/shardingsphere-infra-common/src/test/java/org/apache/shardingsphere/infra/yaml/engine/YamlEngineTest.java
@@ -54,7 +54,7 @@ public final class YamlEngineTest {
BufferedReader reader = new BufferedReader(fileReader)) {
String line;
while (null != (line = reader.readLine())) {
- yamlContent.append(line).append("\n");
+ yamlContent.append(line).append(System.lineSeparator());
}
}
YamlRuleConfigurationFixture actual =
YamlEngine.unmarshal(yamlContent.toString().getBytes(),
YamlRuleConfigurationFixture.class);
@@ -77,7 +77,7 @@ public final class YamlEngineTest {
public void assertMarshal() {
YamlRuleConfigurationFixture actual = new
YamlRuleConfigurationFixture();
actual.setName("test");
- assertThat(YamlEngine.marshal(actual), is("name: test\n"));
+ assertThat(YamlEngine.marshal(actual), is("name: test" +
System.lineSeparator()));
}
@Test(expected = ConstructorException.class)
@@ -90,7 +90,7 @@ public final class YamlEngineTest {
BufferedReader reader = new BufferedReader(fileReader)) {
String line;
while (null != (line = reader.readLine())) {
- yamlContent.append(line).append("\n");
+ yamlContent.append(line).append(System.lineSeparator());
}
}
YamlEngine.unmarshal(yamlContent.toString(),
YamlRootConfiguration.class);
@@ -102,6 +102,9 @@ public final class YamlEngineTest {
actual.setName("test");
YamlRuleConfigurationFixture actualAnother = new
YamlRuleConfigurationFixture();
actualAnother.setName("test");
- assertThat(YamlEngine.marshal(Arrays.asList(actual, actualAnother)),
is("- !FIXTURE\n name: test\n- !FIXTURE\n name: test\n"));
+ StringBuilder res = new StringBuilder("- !FIXTURE");
+ res.append(System.lineSeparator()).append(" name:
test").append(System.lineSeparator()).append("- !FIXTURE")
+ .append(System.lineSeparator()).append(" name:
test").append(System.lineSeparator());
+ assertThat(YamlEngine.marshal(Arrays.asList(actual, actualAnother)),
is(res.toString()));
}
}
diff --git
a/shardingsphere-mode/shardingsphere-mode-core/src/test/java/org/apache/shardingsphere/mode/persist/service/impl/DataSourcePersistServiceTest.java
b/shardingsphere-mode/shardingsphere-mode-core/src/test/java/org/apache/shardingsphere/mode/persist/service/impl/DataSourcePersistServiceTest.java
index 27e748c..212d2b1 100644
---
a/shardingsphere-mode/shardingsphere-mode-core/src/test/java/org/apache/shardingsphere/mode/persist/service/impl/DataSourcePersistServiceTest.java
+++
b/shardingsphere-mode/shardingsphere-mode-core/src/test/java/org/apache/shardingsphere/mode/persist/service/impl/DataSourcePersistServiceTest.java
@@ -81,7 +81,6 @@ public final class DataSourcePersistServiceTest {
@Test
public void assertAppend() {
- when(repository.get("/metadata/foo_db/dataSources")).thenReturn("");
new DataSourcePersistService(repository).append("foo_db",
Collections.singletonMap("foo_ds",
DataSourceConfiguration.getDataSourceConfiguration(createDataSource("foo_ds"))));
String expected =
readDataSourceYaml("yaml/persist/data-source-foo.yaml");
verify(repository).persist("/metadata/foo_db/dataSources", expected);
@@ -92,7 +91,7 @@ public final class DataSourcePersistServiceTest {
String actual =
readDataSourceYaml("yaml/persist/data-source-foo.yaml");
when(repository.get("/metadata/foo_db/dataSources")).thenReturn(actual);
new DataSourcePersistService(repository).drop("foo_db",
Collections.singleton("foo_ds"));
- verify(repository).persist("/metadata/foo_db/dataSources", "{}\n");
+ verify(repository).persist("/metadata/foo_db/dataSources", "{}" +
System.lineSeparator());
}
private DataSource createDataSource(final String name) {
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/api/impl/GovernanceRepositoryAPIImplTest.java
b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/api/impl/GovernanceRepositoryAPIImplTest.java
index 7a75f68..460693e 100644
---
a/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/api/impl/GovernanceRepositoryAPIImplTest.java
+++
b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/api/impl/GovernanceRepositoryAPIImplTest.java
@@ -65,7 +65,7 @@ public final class GovernanceRepositoryAPIImplTest {
JobContext jobContext = mockJobContext();
governanceRepositoryAPI.persistJobProgress(jobContext);
JobProgress actual =
governanceRepositoryAPI.getJobProgress(jobContext.getJobId(),
jobContext.getShardingItem());
- assertThat(actual.toString(), is(mockYamlJobProgress()));
+ assertThat(actual.toString(),
is(ResourceUtil.readFileAndIgnoreComments("governance-repository.yaml")));
}
@Test
@@ -127,18 +127,5 @@ public final class GovernanceRepositoryAPIImplTest {
dumperConfig.setPosition(new PlaceholderPosition());
return ScalingTaskFactory.createIncrementalTask(3, dumperConfig,
taskConfig.getImporterConfig());
}
-
- private String mockYamlJobProgress() {
- return "databaseType: H2\n"
- + "incremental:\n"
- + " ds_0:\n"
- + " delay:\n"
- + " lastEventTimestamps: 0\n"
- + " latestActiveTimeMillis: 0\n"
- + " position: ''\n"
- + "inventory:\n"
- + " unfinished:\n"
- + " ds_0.t_order#0: ''\n"
- + "status: RUNNING\n";
- }
}
+
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/job/progress/JobProgressTest.java
b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/job/progress/JobProgressTest.java
index 58ad7e2..f9693e2 100644
---
a/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/job/progress/JobProgressTest.java
+++
b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/job/progress/JobProgressTest.java
@@ -23,6 +23,7 @@ import
org.apache.shardingsphere.scaling.core.job.position.PlaceholderPosition;
import org.apache.shardingsphere.scaling.core.job.position.PrimaryKeyPosition;
import
org.apache.shardingsphere.scaling.core.job.task.incremental.IncrementalTaskProgress;
import
org.apache.shardingsphere.scaling.core.job.task.inventory.InventoryTaskProgress;
+import org.apache.shardingsphere.scaling.core.util.ResourceUtil;
import org.junit.Test;
import java.util.Collections;
@@ -37,7 +38,7 @@ public final class JobProgressTest {
@Test
public void assertInit() {
- JobProgress jobProgress =
JobProgress.init(mockJobProgressYamlString());
+ JobProgress jobProgress =
JobProgress.init(ResourceUtil.readFileAndIgnoreComments("job-progress.yaml"));
assertThat(jobProgress.getStatus(), is(JobStatus.RUNNING));
assertThat(jobProgress.getDatabaseType(), is("H2"));
assertThat(jobProgress.getInventoryTaskProgressMap().size(), is(4));
@@ -46,13 +47,13 @@ public final class JobProgressTest {
@Test
public void assertGetIncrementalPosition() {
- JobProgress jobProgress =
JobProgress.init(mockJobProgressYamlString());
+ JobProgress jobProgress =
JobProgress.init(ResourceUtil.readFileAndIgnoreComments("job-progress.yaml"));
assertTrue(jobProgress.getIncrementalPosition("ds0") instanceof
PlaceholderPosition);
}
@Test
public void assertGetInventoryPosition() {
- JobProgress jobProgress =
JobProgress.init(mockJobProgressYamlString());
+ JobProgress jobProgress =
JobProgress.init(ResourceUtil.readFileAndIgnoreComments("job-progress.yaml"));
assertThat(jobProgress.getInventoryPosition("ds0").size(), is(2));
assertTrue(jobProgress.getInventoryPosition("ds0").get("ds0.t_1")
instanceof FinishedPosition);
assertTrue(jobProgress.getInventoryPosition("ds1").get("ds1.t_1")
instanceof PlaceholderPosition);
@@ -66,7 +67,7 @@ public final class JobProgressTest {
jobProgress.setDatabaseType("H2");
jobProgress.setIncrementalTaskProgressMap(mockIncrementalTaskProgressMap());
jobProgress.setInventoryTaskProgressMap(mockInventoryTaskProgressMap());
- assertThat(jobProgress.toString(), is(mockJobProgressYamlString()));
+ assertThat(jobProgress.toString(),
is(ResourceUtil.readFileAndIgnoreComments("job-progress.yaml")));
}
private Map<String, IncrementalTaskProgress>
mockIncrementalTaskProgressMap() {
@@ -81,22 +82,4 @@ public final class JobProgressTest {
result.put("ds1.t_2", new InventoryTaskProgress(new
PrimaryKeyPosition(1, 2)));
return result;
}
-
- private String mockJobProgressYamlString() {
- return "databaseType: H2\n"
- + "incremental:\n"
- + " ds0:\n"
- + " delay:\n"
- + " lastEventTimestamps: 0\n"
- + " latestActiveTimeMillis: 0\n"
- + " position: ''\n"
- + "inventory:\n"
- + " finished:\n"
- + " - ds0.t_2\n"
- + " - ds0.t_1\n"
- + " unfinished:\n"
- + " ds1.t_2: 1,2\n"
- + " ds1.t_1: ''\n"
- + "status: RUNNING\n";
- }
}
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/util/ResourceUtil.java
b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/util/ResourceUtil.java
index 3ba8b25..357a003 100644
---
a/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/util/ResourceUtil.java
+++
b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/util/ResourceUtil.java
@@ -19,6 +19,7 @@ package org.apache.shardingsphere.scaling.core.util;
import lombok.SneakyThrows;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.shardingsphere.scaling.core.config.JobConfiguration;
import org.apache.shardingsphere.scaling.core.config.RuleConfiguration;
import
org.apache.shardingsphere.scaling.core.config.datasource.ShardingSphereJDBCDataSourceConfiguration;
@@ -26,7 +27,11 @@ import
org.apache.shardingsphere.scaling.core.config.datasource.StandardJDBCData
import java.io.IOException;
import java.io.InputStream;
+import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.stream.Collectors;
/**
* Resource util.
@@ -76,4 +81,20 @@ public final class ResourceUtil {
return IOUtils.toString(in, StandardCharsets.UTF_8);
}
}
+
+ /**
+ * Ignore comments to read configuration from YAML.
+ *
+ * @return YAML configuration.
+ */
+ /**
+ * Ignore comments to read configuration from YAML.
+ * @param fileName YAML file name.
+ * @return YAML configuration.
+ */
+ @SneakyThrows({IOException.class, URISyntaxException.class})
+ public static String readFileAndIgnoreComments(final String fileName) {
+ return
Files.readAllLines(Paths.get(ClassLoader.getSystemResource(fileName).toURI()))
+ .stream().filter(each -> StringUtils.isNotBlank(each) &&
!each.startsWith("#")).map(each -> each +
System.lineSeparator()).collect(Collectors.joining());
+ }
}
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-core/src/test/resources/governance-repository.yaml
b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/resources/governance-repository.yaml
new file mode 100644
index 0000000..69835bd
--- /dev/null
+++
b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/resources/governance-repository.yaml
@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+databaseType: H2
+incremental:
+ ds_0:
+ delay:
+ lastEventTimestamps: 0
+ latestActiveTimeMillis: 0
+ position: ''
+inventory:
+ unfinished:
+ ds_0.t_order#0: ''
+status: RUNNING
+
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-core/src/test/resources/job-progress.yaml
b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/resources/job-progress.yaml
new file mode 100644
index 0000000..3db934e
--- /dev/null
+++
b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/resources/job-progress.yaml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+databaseType: H2
+incremental:
+ ds0:
+ delay:
+ lastEventTimestamps: 0
+ latestActiveTimeMillis: 0
+ position: ''
+inventory:
+ finished:
+ - ds0.t_2
+ - ds0.t_1
+ unfinished:
+ ds1.t_2: 1,2
+ ds1.t_1: ''
+status: RUNNING
