This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
new 907f9e5b3b [1.1-lts](cherry-pick) Temp partition with spark load and
npm build 14648 and 14629 (#14705)
907f9e5b3b is described below
commit 907f9e5b3b7843751c0be01de4fb27e12891db91
Author: yiguolei <[email protected]>
AuthorDate: Wed Nov 30 18:25:35 2022 +0800
[1.1-lts](cherry-pick) Temp partition with spark load and npm build 14648
and 14629 (#14705)
* [fix](spark load)Temp partition with spark load (#14648)
* [fix](spark load)losing temporary partition item entry
* [fix](spark load)Temp partition with spark load
* [fix](ui) clean npm cache before install (#14629)
npm ERR! Unexpected end of JSON input while parsing near
'...ih/ae0E6HfGdwwO/\r\na'
Co-authored-by: xiaoDjun <[email protected]>
Co-authored-by: Jeffrey <[email protected]>
---
build.sh | 7 ++++---
.../java/org/apache/doris/catalog/RangePartitionInfo.java | 15 +++++++++++++++
.../apache/doris/load/loadv2/SparkLoadPendingTask.java | 2 +-
3 files changed, 20 insertions(+), 4 deletions(-)
diff --git a/build.sh b/build.sh
index df3a632f57..faa55feea1 100755
--- a/build.sh
+++ b/build.sh
@@ -331,9 +331,10 @@ function build_ui() {
if [[ ! -z ${CUSTOM_UI_DIST} ]]; then
ui_dist=${CUSTOM_UI_DIST}
else
- cd ${DORIS_HOME}/ui
- ${NPM} install
- ${NPM} run build
+ cd "${DORIS_HOME}/ui"
+ "${NPM}" cache clean --force
+ "${NPM}" install --legacy-peer-deps
+ "${NPM}" run build
fi
echo "ui dist: ${ui_dist}"
rm -rf ${DORIS_HOME}/fe/fe-core/src/main/resources/static/
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java
index b4e267d19a..a838263d2e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java
@@ -25,6 +25,8 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.util.RangeUtils;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import java.io.DataInput;
@@ -76,6 +78,19 @@ public class RangePartitionInfo extends PartitionInfo {
return itemEntryList;
}
+ public List<Map.Entry<Long, PartitionItem>>
getAllPartitionItemEntryList(boolean isSorted) {
+ Map<Long, PartitionItem> tmpMap = Maps.newHashMap();
+
+ tmpMap.putAll(idToItem);
+ tmpMap.putAll(idToTempItem);
+
+ List<Map.Entry<Long, PartitionItem>> itemEntryList =
Lists.newArrayList(tmpMap.entrySet());
+ if (isSorted) {
+ Collections.sort(itemEntryList,
RangeUtils.RANGE_MAP_ENTRY_COMPARATOR);
+ }
+ return itemEntryList;
+ }
+
// create a new range and check it.
private Range<PartitionKey> createAndCheckNewRange(PartitionKeyDesc
partKeyDesc, boolean isTemp)
throws AnalysisException, DdlException {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java
b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java
index d9f9467a3b..adcbb3edbc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java
@@ -331,7 +331,7 @@ public class SparkLoadPendingTask extends LoadTask {
partitionColumnRefs.add(column.getName());
}
- for (Map.Entry<Long, PartitionItem> entry :
rangePartitionInfo.getPartitionItemEntryList(false, true)) {
+ for (Map.Entry<Long, PartitionItem> entry :
rangePartitionInfo.getAllPartitionItemEntryList(true)) {
long partitionId = entry.getKey();
if (!partitionIds.contains(partitionId)) {
continue;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
