This is an automated email from the ASF dual-hosted git repository.
ssiddiqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new 17ff4ad [MINOR] Fix frame handling in DMVUtils and cleanup for
redundant DML script
17ff4ad is described below
commit 17ff4ad3f2e30830fce3cc0aa4a6dc160d219c71
Author: Shafaq Siddiqi <[email protected]>
AuthorDate: Wed Jun 16 19:47:17 2021 +0200
[MINOR] Fix frame handling in DMVUtils and cleanup for redundant DML script
---
scripts/builtin/getPermutations.dml | 57 ----------------------
.../org/apache/sysds/runtime/util/DMVUtils.java | 24 ++++-----
2 files changed, 13 insertions(+), 68 deletions(-)
diff --git a/scripts/builtin/getPermutations.dml
b/scripts/builtin/getPermutations.dml
deleted file mode 100644
index 460da76..0000000
--- a/scripts/builtin/getPermutations.dml
+++ /dev/null
@@ -1,57 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-# Related to [SYSTEMDS-2662] dependency function for cleaning pipelines
-# This built-in will generate all possible permutation for a given set of
values
-
-
-s_getPermutations = function(Frame[String] opt)
-return(Frame[String] output)
-{
- idx = seq(1, ncol(opt))
- # get the indexes of columns for recode transformation
- index = vectorToCsv(idx)
- # recode logical pipelines for easy handling
- jspecR = "{ids:true, recode:["+index+"]}";
- [X, M] = transformencode(target=opt, spec=jspecR);
- X = replace(target= X, pattern = NaN, replacement = 0)
- # initialize output matrix
- n = nrow(opt)
- d = ncol(opt)
- outC = matrix(0, n^d, d)
-
- parfor(i in 1 : d) {
- # matrix for storing rows of ith columns
- outR = matrix(0, 0, 1)
- j = n^i
- rowIdx = 1
- for(k in 1:j) {
- valDup = matrix(as.scalar(X[rowIdx, i]), n^(d-i), 1)
- outR = rbind(outR, valDup)
- rowIdx = rowIdx + 1
- rowIdx = ifelse(((rowIdx)%%(n+1)) == 0, 1, rowIdx)
- }
- outC[,i] = outR
- }
- computeInvalid = rowMins(outC) == 0
- outC = removeEmpty(target = outC, margin = "rows", select = computeInvalid
== 0)
- output = transformdecode(target=outC, spec=jspecR, meta=M);
-}
\ No newline at end of file
diff --git a/src/main/java/org/apache/sysds/runtime/util/DMVUtils.java
b/src/main/java/org/apache/sysds/runtime/util/DMVUtils.java
index 95ce669..6ab1e1a 100644
--- a/src/main/java/org/apache/sysds/runtime/util/DMVUtils.java
+++ b/src/main/java/org/apache/sysds/runtime/util/DMVUtils.java
@@ -49,10 +49,10 @@ public class DMVUtils {
for (int idx = 0; idx < numCols; idx++) {
Object c = frame.getColumnData(idx);
- String[] column = (String[]) c;
+ String[] attr = (String[]) c;
String key = "";
- for (String attr : column) {
- key = (attr.isEmpty()) ? "NULL": attr;
+ for (int i = 0; i < numRows; i++) {
+ key = (attr[i] == null) ? "NULL": attr[i];
addDistinctValueOrIncrementCounter(table_Hist,
key, idx);
}
}
@@ -278,36 +278,38 @@ public class DMVUtils {
{
int row_idx = -1;
String pattern = "";
- String[] column = (String[]) col;
- for (String attr : column) {
+ String[] attr = (String[]) col;
+ int numRows = frameBlock.getNumRows();
+ for (int i = 0; i < numRows; i++) {
+ String value = (attr[i] == null)? "NULL": attr[i];
switch (level){
case LEVEL1:
- pattern = encodeRawString(attr);
+ pattern = encodeRawString(value);
break;
case LEVEL2:
- pattern = encodeRawString(attr);
+ pattern = encodeRawString(value);
pattern = removeNumbers(pattern);
break;
case LEVEL3:
- pattern = encodeRawString(attr);
+ pattern = encodeRawString(value);
pattern = removeNumbers(pattern);
pattern = removeUpperLowerCase(pattern);
break;
case LEVEL4:
- pattern = encodeRawString(attr);
+ pattern = encodeRawString(value);
pattern = removeNumbers(pattern);
pattern = removeUpperLowerCase(pattern);
pattern =
removeInnerCharacterInPattern(pattern, DIGIT, DOT);
break;
case LEVEL5:
- pattern = encodeRawString(attr);
+ pattern = encodeRawString(value);
pattern = removeNumbers(pattern);
pattern = removeUpperLowerCase(pattern);
pattern =
removeInnerCharacterInPattern(pattern, DIGIT, DOT);
pattern =
removeInnerCharacterInPattern(pattern, ALPHA, SPACE);
break;
case LEVEL6:
- pattern = encodeRawString(attr);
+ pattern = encodeRawString(value);
pattern = removeNumbers(pattern);
pattern = removeUpperLowerCase(pattern);
pattern =
removeInnerCharacterInPattern(pattern, DIGIT, DOT);
