This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new c232ee02239 fix pipe snapshot parser
c232ee02239 is described below
commit c232ee02239e26562d7db27e46e99f586a02c3aa
Author: Colin Li <[email protected]>
AuthorDate: Thu Apr 25 14:15:09 2024 +0800
fix pipe snapshot parser
---
.../consensus/request/ConfigPhysicalPlan.java | 1 +
.../consensus/request/ConfigPhysicalPlanType.java | 1 +
.../iotdb/confignode/persistence/AuthorInfo.java | 3 +
.../confignode/persistence/AuthorInfoTest.java | 21 +++++
.../db/tools/schema/SRStatementGenerator.java | 91 ++++++++++++++--------
.../db/auth/user/LocalFileUserManagerTest.java | 8 ++
.../db/utils/SchemaRegionSnapshotParserTest.java | 86 ++++++++++++++++++--
.../commons/auth/authorizer/BasicAuthorizer.java | 12 ++-
.../iotdb/commons/auth/authorizer/IAuthorizer.java | 2 +
.../iotdb/commons/auth/user/BasicUserManager.java | 16 +++-
.../iotdb/commons/auth/user/IUserManager.java | 3 +-
11 files changed, 200 insertions(+), 44 deletions(-)
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java
index e2c6a5e9632..8ad8c9b2fd2 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java
@@ -312,6 +312,7 @@ public abstract class ConfigPhysicalPlan implements
IConsensusRequest {
case RevokeRole:
case RevokeRoleFromUser:
case UpdateUser:
+ case CreateUserWithRawPassword:
plan = new AuthorPlan(configPhysicalPlanType);
break;
case ApplyConfigNode:
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java
index 4e547304eb9..031518be944 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java
@@ -122,6 +122,7 @@ public enum ConfigPhysicalPlanType {
ListUserRoles((short) 636),
@Deprecated
ListRoleUsers((short) 637),
+ CreateUserWithRawPassword((short) 638),
/** Function. */
CreateFunction((short) 700),
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java
index b026eb43053..f0be01f2b91 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java
@@ -221,6 +221,9 @@ public class AuthorInfo implements SnapshotProcessor {
case CreateUser:
authorizer.createUser(userName, password);
break;
+ case CreateUserWithRawPassword:
+ authorizer.createUserWithRawPassword(userName, password);
+ break;
case CreateRoleDep:
AuthUtils.validateRolenamePre(roleName);
authorizer.createRole(roleName);
diff --git
a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/AuthorInfoTest.java
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/AuthorInfoTest.java
index 4c741d1c361..12bab54572a 100644
---
a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/AuthorInfoTest.java
+++
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/AuthorInfoTest.java
@@ -32,6 +32,7 @@ import
org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;
import org.apache.iotdb.confignode.consensus.request.auth.AuthorPlan;
import org.apache.iotdb.confignode.consensus.response.auth.PermissionInfoResp;
import org.apache.iotdb.confignode.rpc.thrift.TCheckUserPrivilegesReq;
+import org.apache.iotdb.confignode.rpc.thrift.TPermissionInfoResp;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.commons.io.FileUtils;
@@ -1020,4 +1021,24 @@ public class AuthorInfoTest {
}
}
}
+
+ @Test
+ public void createUserWithRawPassword() throws AuthException {
+ TSStatus status;
+ AuthorPlan authorPlan;
+ authorPlan =
+ new AuthorPlan(
+ ConfigPhysicalPlanType.CreateUserWithRawPassword,
+ "testuser",
+ "",
+ AuthUtils.encryptPassword("password"),
+ "",
+ new HashSet<>(),
+ false,
+ new ArrayList<>());
+ status = authorInfo.authorNonQuery(authorPlan);
+ Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(),
status.getCode());
+ TPermissionInfoResp result = authorInfo.login("testuser", "password");
+ Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(),
result.getStatus().getCode());
+ }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/schema/SRStatementGenerator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/schema/SRStatementGenerator.java
index b8b0363db13..5ea6e86dc9c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/schema/SRStatementGenerator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/schema/SRStatementGenerator.java
@@ -280,8 +280,30 @@ public class SRStatementGenerator implements
Iterator<Statement>, Iterable<State
final CreateLogicalViewStatement stmt = new
CreateLogicalViewStatement();
final LogicalViewSchema viewSchema =
(LogicalViewSchema) node.getAsMeasurementMNode().getSchema();
- stmt.setTargetFullPaths(Collections.singletonList(path));
-
stmt.setViewExpressions(Collections.singletonList(viewSchema.getExpression()));
+ if (viewSchema != null) {
+ stmt.setTargetFullPaths(Collections.singletonList(path));
+
stmt.setViewExpressions(Collections.singletonList(viewSchema.getExpression()));
+ return stmt;
+ }
+ // if (node.getOffset() >= 0) {
+ // final AlterTimeSeriesStatement alterTimeSeriesStatement =
new
+ // AlterTimeSeriesStatement();
+ // alterTimeSeriesStatement.setPath(path);
+ // try {
+ // Pair<Map<String, String>, Map<String, String>>
tagsAndAttribute =
+ // getTagsAndAttributes(node.getOffset());
+ // if (tagsAndAttribute != null) {
+ //
alterTimeSeriesStatement.setTagsMap(tagsAndAttribute.left);
+ //
alterTimeSeriesStatement.setAttributesMap(tagsAndAttribute.right);
+ // }
+ // } catch (IOException ioException) {
+ // lastExcept = ioException;
+ // LOGGER.warn(
+ // "Error when parse tag and attributes file of node
path {}", path,
+ // ioException);
+ // }
+ // node.setOffset(0);
+ // }
return stmt;
} else {
final CreateTimeSeriesStatement stmt = new CreateTimeSeriesStatement();
@@ -291,24 +313,18 @@ public class SRStatementGenerator implements
Iterator<Statement>, Iterable<State
stmt.setDataType(node.getDataType());
stmt.setEncoding(node.getAsMeasurementMNode().getSchema().getEncodingType());
if (node.getOffset() >= 0) {
- if (tagFileChannel != null) {
- try {
- final ByteBuffer byteBuffer =
-
ByteBuffer.allocate(COMMON_CONFIG.getTagAttributeTotalSize());
- tagFileChannel.read(byteBuffer, node.getOffset());
- byteBuffer.flip();
- final Pair<Map<String, String>, Map<String, String>>
tagsAndAttributes =
- new Pair<>(
- ReadWriteIOUtils.readMap(byteBuffer),
ReadWriteIOUtils.readMap(byteBuffer));
+ try {
+ final Pair<Map<String, String>, Map<String, String>>
tagsAndAttributes =
+ getTagsAndAttributes(node.getOffset());
+ if (tagsAndAttributes != null) {
stmt.setTags(tagsAndAttributes.left);
stmt.setAttributes(tagsAndAttributes.right);
- } catch (IOException exception) {
- lastExcept = exception;
- LOGGER.warn("Error when parser tag and attributes files",
exception);
}
- } else {
- LOGGER.warn("Timeseries has attributes and tags but don't find tag
file");
+ } catch (IOException ioException) {
+ lastExcept = ioException;
+ LOGGER.warn("Error when parser tag and attributes files",
ioException);
}
+ node.setOffset(0);
}
return stmt;
}
@@ -338,30 +354,41 @@ public class SRStatementGenerator implements
Iterator<Statement>, Iterable<State
stmt.addEncoding(measurement.getAsMeasurementMNode().getSchema().getEncodingType());
stmt.addCompressor(measurement.getAsMeasurementMNode().getSchema().getCompressor());
if (measurement.getAsMeasurementMNode().getOffset() >= 0) {
- if (tagFileChannel != null) {
- try {
- ByteBuffer byteBuffer =
ByteBuffer.allocate(COMMON_CONFIG.getTagAttributeTotalSize());
- tagFileChannel.read(byteBuffer,
measurement.getAsMeasurementMNode().getOffset());
- byteBuffer.flip();
- Pair<Map<String, String>, Map<String, String>> tagsAndAttributes
=
- new Pair<>(
- ReadWriteIOUtils.readMap(byteBuffer),
ReadWriteIOUtils.readMap(byteBuffer));
+ try {
+ Pair<Map<String, String>, Map<String, String>> tagsAndAttributes =
+
getTagsAndAttributes(measurement.getAsMeasurementMNode().getOffset());
+ if (tagsAndAttributes != null) {
stmt.addAttributesList(tagsAndAttributes.right);
stmt.addTagsList(tagsAndAttributes.left);
- } catch (IOException exception) {
- lastExcept = exception;
- LOGGER.warn(
- "Error when parse tag and attributes file of node path {}",
- measurement.getPartialPath(),
- exception);
}
- } else {
- LOGGER.warn("Measurement has set attributes or tags, but not find
snapshot files");
+ } catch (IOException ioException) {
+ lastExcept = ioException;
+ LOGGER.warn(
+ "Error when parse tag and attributes file of node path {}",
path, ioException);
}
+ measurement.getAsMeasurementMNode().setOffset(0);
+ } else {
+ stmt.addAttributesList(null);
+ stmt.addTagsList(null);
}
}
return stmt;
}
return null;
}
+
+ private Pair<Map<String, String>, Map<String, String>>
getTagsAndAttributes(long offset)
+ throws IOException {
+ if (tagFileChannel != null) {
+ ByteBuffer byteBuffer =
ByteBuffer.allocate(COMMON_CONFIG.getTagAttributeTotalSize());
+ tagFileChannel.read(byteBuffer, offset);
+ byteBuffer.flip();
+ Pair<Map<String, String>, Map<String, String>> tagsAndAttributes =
+ new Pair<>(ReadWriteIOUtils.readMap(byteBuffer),
ReadWriteIOUtils.readMap(byteBuffer));
+ return tagsAndAttributes;
+ } else {
+ LOGGER.warn("Measurement has set attributes or tags, but not find
snapshot files");
+ }
+ return null;
+ }
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/user/LocalFileUserManagerTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/user/LocalFileUserManagerTest.java
index a837df4c654..43cd6df68dd 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/user/LocalFileUserManagerTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/user/LocalFileUserManagerTest.java
@@ -183,4 +183,12 @@ public class LocalFileUserManagerTest {
assertEquals(users[i].getName(), usernames.get(i + 1));
}
}
+
+ @Test
+ public void testCreateUserRawPassword() throws AuthException {
+ Assert.assertTrue(
+ manager.createUser("testRaw", AuthUtils.encryptPassword("password1"),
true, false));
+ User user = manager.getUser("testRaw");
+ Assert.assertEquals(user.getPassword(),
AuthUtils.encryptPassword("password1"));
+ }
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaRegionSnapshotParserTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaRegionSnapshotParserTest.java
index e893ab75cae..8ff09c5008b 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaRegionSnapshotParserTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaRegionSnapshotParserTest.java
@@ -254,32 +254,43 @@ public class SchemaRegionSnapshotParserTest {
ICreateAlignedTimeSeriesPlan plan =
SchemaRegionWritePlanFactory.getCreateAlignedTimeSeriesPlan(
new PartialPath("root.sg.t1.t2"),
- Arrays.asList("s1", "s2"),
- Arrays.asList(TSDataType.INT32, TSDataType.INT64),
- Arrays.asList(TSEncoding.PLAIN, TSEncoding.RLE),
- Arrays.asList(CompressionType.SNAPPY, CompressionType.LZ4),
- Arrays.asList("alias1", "alias2"),
+ Arrays.asList("s1", "s2", "s3"),
+ Arrays.asList(TSDataType.INT32, TSDataType.INT64,
TSDataType.BOOLEAN),
+ Arrays.asList(TSEncoding.PLAIN, TSEncoding.RLE, TSEncoding.PLAIN),
+ Arrays.asList(
+ CompressionType.SNAPPY, CompressionType.LZ4,
CompressionType.UNCOMPRESSED),
+ Arrays.asList("alias1", "alias2", null),
Arrays.asList(
new HashMap<String, String>() {
{
put("tag1", "t1");
+ put("tag_1", "value2");
}
},
new HashMap<String, String>() {
{
put("tag2", "t2");
+ put("tag_2", "t_2");
}
- }),
+ },
+ new HashMap<>()), // not all measurements have tag
Arrays.asList(
new HashMap<String, String>() {
{
put("attr1", "a1");
+ put("attr_1", "a_1");
}
},
new HashMap<String, String>() {
{
put("attr2", "a2");
}
+ },
+ new HashMap<String, String>() {
+ {
+ put("tag2", "t2");
+ put("tag_2", "t_2");
+ }
}));
schemaRegion.createAlignedTimeSeries(plan);
File snapshotDir = new File(config.getSchemaDir() + File.separator +
"snapshot");
@@ -305,6 +316,69 @@ public class SchemaRegionSnapshotParserTest {
CreateAlignedTimeSeriesStatement createAlignedTimeSeriesStatement =
(CreateAlignedTimeSeriesStatement) stmt;
Assert.assertEquals(plan.getDevicePath(),
createAlignedTimeSeriesStatement.getDevicePath());
+
+ Assert.assertEquals(
+ plan.getTagsList().size(),
createAlignedTimeSeriesStatement.getTagsList().size());
+ Assert.assertEquals(
+ plan.getAliasList().size(),
createAlignedTimeSeriesStatement.getAliasList().size());
+ Assert.assertEquals(
+ plan.getAttributesList().size(),
+ createAlignedTimeSeriesStatement.getAttributesList().size());
+ Assert.assertEquals(
+ createAlignedTimeSeriesStatement.getMeasurements().size(),
+ createAlignedTimeSeriesStatement.getAttributesList().size());
+ Assert.assertEquals(
+ createAlignedTimeSeriesStatement.getMeasurements().size(),
+ createAlignedTimeSeriesStatement.getTagsList().size());
+
+ Comparator<Map<String, String>> comp =
+ new Comparator<Map<String, String>>() {
+ @Override
+ public int compare(Map<String, String> o1, Map<String, String> o2)
{
+ if (o1 == null && o2 == null) {
+ return 0;
+ }
+ if (o1 == null) {
+ return -1;
+ }
+ if (o2 == null) {
+ return 1;
+ }
+ return Integer.compare(o1.hashCode(), o2.hashCode());
+ }
+ };
+
+ Comparator<String> comp_str =
+ new Comparator<String>() {
+ @Override
+ public int compare(String o1, String o2) {
+ if (o1 == null && o2 == null) {
+ return 0;
+ }
+ if (o1 == null) {
+ return -1;
+ }
+ if (o2 == null) {
+ return 1;
+ }
+ return Integer.compare(o1.hashCode(), o2.hashCode());
+ }
+ };
+
+ Collections.sort(plan.getAliasList(), comp_str);
+ Collections.sort(createAlignedTimeSeriesStatement.getAliasList(),
comp_str);
+ Collections.sort(plan.getAttributesList(), comp);
+ Collections.sort(createAlignedTimeSeriesStatement.getAttributesList(),
comp);
+ Collections.sort(plan.getMeasurements(), comp_str);
+ Collections.sort(createAlignedTimeSeriesStatement.getMeasurements(),
comp_str);
+ Collections.sort(plan.getTagsList(), comp);
+ Collections.sort(createAlignedTimeSeriesStatement.getTagsList(), comp);
+ Collections.sort(plan.getEncodings());
+ Collections.sort(createAlignedTimeSeriesStatement.getEncodings());
+ Collections.sort(plan.getCompressors());
+ Collections.sort(createAlignedTimeSeriesStatement.getCompressors());
+ Collections.sort(plan.getDataTypes());
+ Collections.sort(createAlignedTimeSeriesStatement.getDataTypes());
Assert.assertEquals(
plan.getMeasurements(),
createAlignedTimeSeriesStatement.getMeasurements());
Assert.assertEquals(plan.getAliasList(),
createAlignedTimeSeriesStatement.getAliasList());
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java
index 4852ceee3e8..e8359bcd070 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java
@@ -125,7 +125,7 @@ public abstract class BasicAuthorizer implements
IAuthorizer, IService {
@Override
public void createUser(String username, String password) throws
AuthException {
- if (!userManager.createUser(username, password, true)) {
+ if (!userManager.createUser(username, password, true, true)) {
throw new AuthException(
TSStatusCode.USER_ALREADY_EXIST, String.format("User %s already
exists", username));
}
@@ -133,7 +133,15 @@ public abstract class BasicAuthorizer implements
IAuthorizer, IService {
@Override
public void createUserWithoutCheck(String username, String password) throws
AuthException {
- if (!userManager.createUser(username, password, false)) {
+ if (!userManager.createUser(username, password, false, true)) {
+ throw new AuthException(
+ TSStatusCode.USER_ALREADY_EXIST, String.format("User %s already
exists", username));
+ }
+ }
+
+ @Override
+ public void createUserWithRawPassword(String username, String password)
throws AuthException {
+ if (!userManager.createUser(username, password, true, false)) {
throw new AuthException(
TSStatusCode.USER_ALREADY_EXIST, String.format("User %s already
exists", username));
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java
index 7546224bb76..8b6d81a40f1 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java
@@ -284,5 +284,7 @@ public interface IAuthorizer extends SnapshotProcessor {
*/
void createUserWithoutCheck(String username, String password) throws
AuthException;
+ void createUserWithRawPassword(String username, String password) throws
AuthException;
+
void checkUserPathPrivilege();
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/BasicUserManager.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/BasicUserManager.java
index aeeac635e24..b6445019d84 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/BasicUserManager.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/BasicUserManager.java
@@ -103,6 +103,7 @@ public abstract class BasicUserManager implements
IUserManager {
createUser(
CommonDescriptor.getInstance().getConfig().getAdminName(),
CommonDescriptor.getInstance().getConfig().getAdminPassword(),
+ true,
true);
setUserUseWaterMark(CommonDescriptor.getInstance().getConfig().getAdminName(),
false);
}
@@ -138,11 +139,14 @@ public abstract class BasicUserManager implements
IUserManager {
}
@Override
- public boolean createUser(String username, String password, boolean
validCheck)
+ public boolean createUser(
+ String username, String password, boolean validCheck, boolean
enableEncrypt)
throws AuthException {
if (validCheck) {
AuthUtils.validateUsername(username);
- AuthUtils.validatePassword(password);
+ if (enableEncrypt) {
+ AuthUtils.validatePassword(password);
+ }
}
User user = getUser(username);
@@ -151,7 +155,7 @@ public abstract class BasicUserManager implements
IUserManager {
}
lock.writeLock(username);
try {
- user = new User(username, AuthUtils.encryptPassword(password));
+ user = new User(username, enableEncrypt ?
AuthUtils.encryptPassword(password) : password);
userMap.put(username, user);
return true;
} finally {
@@ -389,4 +393,10 @@ public abstract class BasicUserManager implements
IUserManager {
AuthUtils.checkAndRefreshPri(user);
});
}
+
+ @TestOnly
+ public boolean createUser(String username, String password, boolean
validCheck)
+ throws AuthException {
+ return createUser(username, password, validCheck, true);
+ }
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/IUserManager.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/IUserManager.java
index e1f830e6fa7..d33f320a406 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/IUserManager.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/user/IUserManager.java
@@ -47,7 +47,8 @@ public interface IUserManager extends SnapshotProcessor {
* @return True if the user is successfully created, false when the user
already exists.
* @throws AuthException if the given username or password is illegal.
*/
- boolean createUser(String username, String password, boolean validCheck)
throws AuthException;
+ boolean createUser(String username, String password, boolean validCheck,
boolean enableEncrypt)
+ throws AuthException;
/**
* Delete a user.
