Hi i have coded something like this , pls tell me how bad it is .

package Spark.spark;
import java.util.List;
import java.util.function.Function;

import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SQLContext;



public class App 
{
        static long counter=1;
    public static void main( String[] args )
    {
        
        
        
        SparkConf conf = new
SparkConf().setAppName("sorter").setMaster("local[2]").set("spark.executor.memory","1g");
        JavaSparkContext sc = new JavaSparkContext(conf);
        
        SQLContext sqlContext = new org.apache.spark.sql.SQLContext(sc);
        
        DataFrame df = sqlContext.read().json("path");
        DataFrame sortedDF = df.sort("id");
        //df.show();
        //sortedDF.printSchema();
        
        System.out.println(sortedDF.collectAsList().toString());
        JavaRDD<Row> distData = sc.parallelize(sortedDF.collectAsList());
        
        
     List<String >missingNumbers=distData.map(new
org.apache.spark.api.java.function.Function<Row, String>() {
           

                        public String call(Row arg0) throws Exception {
                                // TODO Auto-generated method stub
                                
                                
                                if(counter!=new 
Integer(arg0.getString(0)).intValue())
                                {
                                        StringBuffer misses = new 
StringBuffer();
                                        long newCounter=counter;
                                        while(newCounter!=new 
Integer(arg0.getString(0)).intValue())
                                        {
                                                misses.append(new String(new 
Integer((int) counter).toString()) );
                                                newCounter++;
                                                
                                        }
                                        counter=new 
Integer(arg0.getString(0)).intValue()+1;
                                        return misses.toString();
                                        
                                }
                                counter++;
                                return null;
                                
                        
                                
                        }
                }).collect();
        
        
        
            for (String name: missingNumbers) {
                      System.out.println(name);
                    }
        
        
        
    }
}




--
View this message in context: 
http://apache-spark-user-list.1001560.n3.nabble.com/filling-missing-values-in-a-sequence-tp5708p27748.html
Sent from the Apache Spark User List mailing list archive at Nabble.com.

---------------------------------------------------------------------
To unsubscribe e-mail: user-unsubscr...@spark.apache.org

Reply via email to