BIGTOP-1991. Add BigTop Weatherman
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/ad88e741 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/ad88e741 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/ad88e741 Branch: refs/heads/master Commit: ad88e7414540d6631854f1a731a3c1c029a3c965 Parents: be796c9 Author: RJ Nowling <[email protected]> Authored: Tue Sep 15 09:46:46 2015 -0500 Committer: RJ Nowling <[email protected]> Committed: Tue Sep 15 09:46:46 2015 -0500 ---------------------------------------------------------------------- bigtop-data-generators/README.md | 5 + .../bigpetstore-data-generator/README.md | 34 +- .../bigpetstore-data-generator/build.gradle | 60 +- .../bigtop-location-data/README.md | 26 +- .../bigtop-location-data/build.gradle | 49 +- .../datagenerators/locations/Location.java | 136 +-- .../locations/LocationConstants.java | 10 +- .../locations/LocationReader.java | 282 +++--- .../bigtop-name-generator/README.md | 26 +- .../bigtop-name-generator/build.gradle | 50 +- .../bigtop-samplers/README.md | 26 +- .../bigtop-samplers/build.gradle | 51 +- .../samplers/samplers/GammaSampler.java | 34 + .../bigtop-weatherman/README.md | 36 + .../bigtop-weatherman/build.gradle | 37 + .../bigtop-weatherman/settings.gradle | 16 + .../weatherman/WeatherGenerator.java | 67 ++ .../weatherman/WeatherRecord.java | 77 ++ .../weatherman/internal/Driver.java | 194 ++++ .../internal/PrecipitationSampler.java | 51 ++ .../weatherman/internal/TemperatureSampler.java | 71 ++ .../weatherman/internal/WeatherConstants.java | 53 ++ .../internal/WeatherParametersReader.java | 80 ++ .../internal/WeatherRecordBuilder.java | 163 ++++ .../weatherman/internal/WeatherSampler.java | 50 ++ .../internal/WeatherSamplerBuilder.java | 80 ++ .../internal/WeatherStationParameters.java | 110 +++ .../weatherman/internal/WindSpeedSampler.java | 74 ++ .../resources/input_data/weather_parameters.csv | 897 +++++++++++++++++++ bigtop-data-generators/build.gradle | 44 + bigtop-data-generators/settings.gradle | 2 +- 31 files changed, 2405 insertions(+), 486 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/README.md ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/README.md b/bigtop-data-generators/README.md index 4765662..e6fd5c7 100644 --- a/bigtop-data-generators/README.md +++ b/bigtop-data-generators/README.md @@ -24,6 +24,7 @@ The following data generators are included so far: * BigPetStore -- generates customers, stores, products, and transactions for a fictional chain of pet stores * BigTop Name Generator -- generates names by sampling from U.S. Census data +* BigTop Weatherman -- weather simulator We have the following libraries: @@ -45,3 +46,7 @@ Jar files can be installed to a local Maven cache to simplify integration by ext $ gradle install You can then define the dependencies via Maven. + +Running +------- +Please see READMEs in individual project directories. http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigpetstore-data-generator/README.md ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigpetstore-data-generator/README.md b/bigtop-data-generators/bigpetstore-data-generator/README.md index 1acfe90..9e65db6 100644 --- a/bigtop-data-generators/bigpetstore-data-generator/README.md +++ b/bigtop-data-generators/bigpetstore-data-generator/README.md @@ -17,35 +17,11 @@ BigPetStore Data Generator ========================== -BigPetStore ... +Library for simulating customer purchasing behavior at a fictional chain of petstores for the purpose of generating synthetic transaction data. -Data Generator ... - -======= Building and Testing -------------------- -We use the Gradle build system for the BPS data generator so you'll need -to install Gradle on your system. -Once that's done, you can use gradle to run the included unit tests -and build the data generator jar. - -To build: - - $ gradle build - -This will create several directories and a jar located at: - - build/libs/bigpetstore-data-generator-0.9.0-SNAPSHOT.jar - -Building automatically runs the included unit tests. If you would prefer -to just run the unit tests, you can do so by: - - $ gradle test - - -To clean up the build files, run: - - $ gradle clean +The data generator is part of a Gradle multiproject build. Please see the README in the parent directory for build and test instructions. Running the Data Generator @@ -55,12 +31,12 @@ Hadoop or Spark applications) or using a command-line interface. The data generator CLI requires several parameters. To get descriptions: - $ java -jar build/libs/bigpetstore-data-generator-0.9.0-SNAPSHOT.jar + $ java -jar build/libs/bigpetstore-data-generator-1.1.0-SNAPSHOT.jar Here is an example for generating 10 stores, 1000 customers, 100 purchasing models, and a year of transactions: - $ java -jar build/libs/bigpetstore-data-generator-0.9.0-SNAPSHOT.jar generatedData/ 10 1000 100 365.0 + $ java -jar build/libs/bigpetstore-data-generator-1.1.0-SNAPSHOT.jar generatedData/ 10 1000 100 365.0 Groovy Drivers for Scripting @@ -75,5 +51,5 @@ To use Groovy scripts, you will need to have Groovy installed on your system. B data generator as instructed above. Then run the scripts in the `groovy_example_drivers` directory as so: - $ groovy -classpath ../build/libs/bigpetstore-data-generator-0.9.0-SNAPSHOT.jar MonteCarloExponentialSamplingExample.groovy + $ groovy -classpath ../build/libs/bigpetstore-data-generator-1.1.0-SNAPSHOT.jar MonteCarloExponentialSamplingExample.groovy http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigpetstore-data-generator/build.gradle ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigpetstore-data-generator/build.gradle b/bigtop-data-generators/bigpetstore-data-generator/build.gradle index 9787512..2d5b4cd 100644 --- a/bigtop-data-generators/bigpetstore-data-generator/build.gradle +++ b/bigtop-data-generators/bigpetstore-data-generator/build.gradle @@ -13,55 +13,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -apply plugin: 'eclipse' -apply plugin: 'groovy' -apply plugin: 'java' -apply plugin: 'maven' -group = 'org.apache.bigtop' -version = '1.1.0-SNAPSHOT' jar { - - from { - configurations.runtime.collect { - it.isDirectory() ? it : zipTree(it) - } - } - - manifest { - attributes 'Title': 'BigPetStore Data Generator', 'Version': version - attributes 'Main-Class': 'org.apache.bigtop.datagenerators.bigpetstore.cli.Driver' + from { + configurations.runtime.collect { + it.isDirectory() ? it : zipTree(it) } -} - -repositories { - mavenLocal() - mavenCentral() -} - -test { - // show standard out and error on console - testLogging.showStandardStreams = true - - // listen to events in the test execution lifecycle - beforeTest { descriptor -> - logger.lifecycle("Running test: " + descriptor) - } - - // listen to standard out and standard error of the test JVM(s) - onOutput { descriptor, event -> - logger.lifecycle("Test: " + descriptor + " produced standard out/err: " + event.message ) - } + } + manifest { + attributes 'Title': 'BigPetStore Data Generator', 'Version': version + attributes 'Main-Class': 'org.apache.bigtop.datagenerators.bigpetstore.cli.Driver' + } } dependencies { - compile 'com.google.guava:guava:18.0' - compile 'com.google.code.gson:gson:2.3' - compile 'org.apache.commons:commons-lang3:3.4' - compile project(":bigtop-samplers") - compile project(":bigtop-name-generator") - compile project(":bigtop-location-data") - - testCompile 'junit:junit:4.+' + compile 'com.google.guava:guava:18.0' + compile 'com.google.code.gson:gson:2.3' + compile 'org.apache.commons:commons-lang3:3.4' + compile project(":bigtop-samplers") + compile project(":bigtop-name-generator") + compile project(":bigtop-location-data") + + testCompile 'junit:junit:4.+' } http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-location-data/README.md ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-location-data/README.md b/bigtop-data-generators/bigtop-location-data/README.md index 9aff06f..b61eeb0 100644 --- a/bigtop-data-generators/bigtop-location-data/README.md +++ b/bigtop-data-generators/bigtop-location-data/README.md @@ -23,28 +23,4 @@ data model. Building and Testing -------------------- -We use the Gradle build system for the BPS data generator so you'll need -to install Gradle on your system. -Once that's done, you can use gradle to run the included unit tests -and build the data generator jar. - -To build: - - $ gradle build - -This will create several directories and a jar located at: - - build/libs/bigtop-location-data-1.1.0-SNAPSHOT.jar - -Building automatically runs the included unit tests. If you would prefer -to just run the unit tests, you can do so by: - - $ gradle test - -To clean up the build files, run: - - $ gradle clean - -To install a jar into your local maven repository: - - $ gradle install +This project is part of a Gradle multi-project build -- please see the parent directory for build instructions. http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-location-data/build.gradle ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-location-data/build.gradle b/bigtop-data-generators/bigtop-location-data/build.gradle index 9eb91e3..bbced98 100644 --- a/bigtop-data-generators/bigtop-location-data/build.gradle +++ b/bigtop-data-generators/bigtop-location-data/build.gradle @@ -13,51 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -apply plugin: 'eclipse' -apply plugin: 'groovy' -apply plugin: 'java' -apply plugin: 'maven' -group = 'org.apache.bigtop' -version = '1.1.0-SNAPSHOT' jar { - - from { - configurations.runtime.collect { - it.isDirectory() ? it : zipTree(it) - } - } - - manifest { - attributes 'Title': 'BigTop Samplers', 'Version': version + from { + configurations.runtime.collect { + it.isDirectory() ? it : zipTree(it) } -} - -repositories { - mavenLocal() - mavenCentral() -} - -test { - // show standard out and error on console - testLogging.showStandardStreams = true - - // listen to events in the test execution lifecycle - beforeTest { descriptor -> - logger.lifecycle("Running test: " + descriptor) - } - - // listen to standard out and standard error of the test JVM(s) - onOutput { descriptor, event -> - logger.lifecycle("Test: " + descriptor + " produced standard out/err: " + event.message ) - } + } + manifest { + attributes 'Title': 'BigTop Samplers', 'Version': version + } } dependencies { - compile 'com.google.guava:guava:18.0' - - compile 'org.apache.commons:commons-lang3:3.4' + compile 'com.google.guava:guava:18.0' + compile 'org.apache.commons:commons-lang3:3.4' - testCompile 'junit:junit:4.+' + testCompile 'junit:junit:4.+' } http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/Location.java ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/Location.java b/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/Location.java index 62afc0a..58f6cdf 100644 --- a/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/Location.java +++ b/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/Location.java @@ -19,72 +19,72 @@ import java.io.Serializable; import org.apache.commons.lang3.tuple.Pair; -public class Location implements Serializable -{ - private static final long serialVersionUID = 1769986686070108470L; - - final String zipcode; - final Pair<Double, Double> coordinates; - final String city; - final String state; - final double medianHouseholdIncome; - final long population; - - public Location(String zipcode, Pair<Double, Double> coordinates, - String city, String state, double medianHouseholdIncome, long population) - { - this.city = city; - this.state = state; - this.zipcode = zipcode; - this.coordinates = coordinates; - this.medianHouseholdIncome = medianHouseholdIncome; - this.population = population; - } - - public String getZipcode() - { - return zipcode; - } - - public Pair<Double, Double> getCoordinates() - { - return coordinates; - } - - public double getMedianHouseholdIncome() - { - return medianHouseholdIncome; - } - - public long getPopulation() - { - return population; - } - - public double distance(Location other) - { - if(other.getZipcode().equals(zipcode)) - return 0.0; - - Pair<Double, Double> otherCoords = other.getCoordinates(); - - double dist = Math.sin(Math.toRadians(coordinates.getLeft())) * - Math.sin(Math.toRadians(otherCoords.getLeft())) + - Math.cos(Math.toRadians(coordinates.getLeft())) * - Math.cos(Math.toRadians(otherCoords.getLeft())) * - Math.cos(Math.toRadians(coordinates.getRight() - otherCoords.getRight())); - dist = Math.toDegrees(Math.acos(dist)) * 69.09; - - return dist; - } - - public String getCity() - { - return city; - } - - public String getState() - { - return state; - } +public class Location implements Serializable { + private static final long serialVersionUID = 1769986686070108470L; + + final String zipcode; + final Pair<Double, Double> coordinates; + final String city; + final String state; + final double medianHouseholdIncome; + final long population; + + public Location(String zipcode, Pair<Double, Double> coordinates, String city, + String state, double medianHouseholdIncome, long population) { + this.city = city; + this.state = state; + this.zipcode = zipcode; + this.coordinates = coordinates; + this.medianHouseholdIncome = medianHouseholdIncome; + this.population = population; + } + + public String getZipcode() { + return zipcode; + } + + public Pair<Double, Double> getCoordinates() { + return coordinates; + } + + public double getMedianHouseholdIncome() { + return medianHouseholdIncome; + } + + public long getPopulation() { + return population; + } + + public double distance(Pair<Double, Double> otherCoords) { + if (Math.abs(coordinates.getLeft() - otherCoords.getLeft()) < 1e-5 + || Math.abs(coordinates.getRight() - otherCoords.getRight()) < 1e-5) + return 0.0; + + double dist = Math.sin(Math.toRadians(coordinates.getLeft())) + * Math.sin(Math.toRadians(otherCoords.getLeft())) + + Math.cos(Math.toRadians(coordinates.getLeft())) + * Math.cos(Math.toRadians(otherCoords.getLeft())) + * Math.cos(Math.toRadians( + coordinates.getRight() - otherCoords.getRight())); + dist = Math.toDegrees(Math.acos(dist)) * 69.09; + + return dist; + } + + public double distance(Location other) { + if (other.getZipcode().equals(zipcode)) + return 0.0; + + Pair<Double, Double> otherCoords = other.getCoordinates(); + + return distance(otherCoords); + } + + public String getCity() { + return city; + } + + public String getState() { + return state; + } } http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationConstants.java ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationConstants.java b/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationConstants.java index 4140bfd..e6ea242 100644 --- a/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationConstants.java +++ b/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationConstants.java @@ -17,9 +17,9 @@ package org.apache.bigtop.datagenerators.locations; import java.io.File; -public class LocationConstants -{ - public static final File COORDINATES_FILE = new File("zips.csv"); - public static final File INCOMES_FILE = new File("ACS_12_5YR_S1903/ACS_12_5YR_S1903_with_ann.csv"); - public static final File POPULATION_FILE = new File("population_data.csv"); +public class LocationConstants { + public static final File COORDINATES_FILE = new File("zips.csv"); + public static final File INCOMES_FILE = new File( + "ACS_12_5YR_S1903/ACS_12_5YR_S1903_with_ann.csv"); + public static final File POPULATION_FILE = new File("population_data.csv"); } http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationReader.java ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationReader.java b/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationReader.java index 5be4d59..6d5b29e 100644 --- a/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationReader.java +++ b/bigtop-data-generators/bigtop-location-data/src/main/java/org/apache/bigtop/datagenerators/locations/LocationReader.java @@ -32,151 +32,141 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -public class LocationReader -{ - private static class ZipcodeLocationRecord - { - public final Pair<Double, Double> coordinates; - public final String state; - public final String city; - - public ZipcodeLocationRecord(Pair<Double, Double> coordinates, - String city, String state) - { - this.coordinates = coordinates; - this.city = city; - this.state = state; - } - } - - private InputStream getResource(File filename) - { - InputStream stream = getClass().getResourceAsStream("/input_data/" + filename); - return new BufferedInputStream(stream); - } - - private ImmutableMap<String, Double> readIncomeData(InputStream path) throws FileNotFoundException - { - Scanner scanner = new Scanner(path); - - // skip headers - scanner.nextLine(); - scanner.nextLine(); - - Map<String, Double> entries = Maps.newHashMap(); - while(scanner.hasNextLine()) - { - String line = scanner.nextLine().trim(); - String[] cols = line.split(","); - // zipcodes are in the form "ZCTA5 XXXXX" - String zipcode = cols[2].split(" ")[1].trim(); - try - { - double medianHouseholdIncome = Integer.parseInt(cols[5].trim()); - entries.put(zipcode, medianHouseholdIncome); - } - catch(NumberFormatException e) - { - - } - } - - scanner.close(); - - return ImmutableMap.copyOf(entries); - } - - private ImmutableMap<String, Long> readPopulationData(InputStream path) throws FileNotFoundException - { - Scanner scanner = new Scanner(path); - - // skip header - scanner.nextLine(); - - Map<String, Long> entries = Maps.newHashMap(); - while(scanner.hasNextLine()) - { - String line = scanner.nextLine().trim(); - - if(line.length() == 0) - continue; - - String[] cols = line.split(","); - - String zipcode = cols[0].trim(); - Long population = Long.parseLong(cols[1].trim()); - - if(entries.containsKey(zipcode)) - { - entries.put(zipcode, Math.max(entries.get(zipcode), population)); - } - else - { - entries.put(zipcode, population); - } - } - - scanner.close(); - - return ImmutableMap.copyOf(entries); - } - - private ImmutableMap<String, ZipcodeLocationRecord> readCoordinates(InputStream path) throws FileNotFoundException - { - Scanner scanner = new Scanner(path); - - // skip header - scanner.nextLine(); - - Map<String, ZipcodeLocationRecord> entries = Maps.newHashMap(); - while(scanner.hasNextLine()) - { - String line = scanner.nextLine().trim(); - - String[] cols = line.split(", "); - - // remove quote marks - String zipcode = cols[0].substring(1, cols[0].length() - 1); - String state = cols[1].substring(1, cols[1].length() - 1); - Double latitude = Double.parseDouble(cols[2].substring(1, cols[2].length() - 1)); - Double longitude = Double.parseDouble(cols[3].substring(1, cols[3].length() - 1)); - String city = cols[4].substring(1, cols[4].length() - 1); - - Pair<Double, Double> coords = Pair.of(latitude, longitude); - - ZipcodeLocationRecord record = new ZipcodeLocationRecord(coords, city, state); - - entries.put(zipcode, record); - } - - scanner.close(); - - return ImmutableMap.copyOf(entries); - } - - public ImmutableList<Location> readData() throws FileNotFoundException - { - - ImmutableMap<String, Double> incomes = readIncomeData(getResource(LocationConstants.INCOMES_FILE)); - ImmutableMap<String, Long> populations = readPopulationData(getResource(LocationConstants.POPULATION_FILE)); - ImmutableMap<String, ZipcodeLocationRecord> coordinates = readCoordinates(getResource(LocationConstants.COORDINATES_FILE)); - - Set<String> zipcodeSubset = new HashSet<String>(incomes.keySet()); - zipcodeSubset.retainAll(populations.keySet()); - zipcodeSubset.retainAll(coordinates.keySet()); - - List<Location> table = new Vector<Location>(); - for(String zipcode : zipcodeSubset) - { - Location record = new Location(zipcode, - coordinates.get(zipcode).coordinates, - coordinates.get(zipcode).city, - coordinates.get(zipcode).state, - incomes.get(zipcode), - populations.get(zipcode)); - table.add(record); - } +public class LocationReader { + private static class ZipcodeLocationRecord { + public final Pair<Double, Double> coordinates; + public final String state; + public final String city; + + public ZipcodeLocationRecord(Pair<Double, Double> coordinates, String city, + String state) { + this.coordinates = coordinates; + this.city = city; + this.state = state; + } + } - return ImmutableList.copyOf(table); - } + private InputStream getResource(File filename) { + InputStream stream = getClass() + .getResourceAsStream("/input_data/" + filename); + return new BufferedInputStream(stream); + } + + private ImmutableMap<String, Double> readIncomeData(InputStream path) + throws FileNotFoundException { + Scanner scanner = new Scanner(path); + + // skip headers + scanner.nextLine(); + scanner.nextLine(); + + Map<String, Double> entries = Maps.newHashMap(); + while (scanner.hasNextLine()) { + String line = scanner.nextLine().trim(); + String[] cols = line.split(","); + // zipcodes are in the form "ZCTA5 XXXXX" + String zipcode = cols[2].split(" ")[1].trim(); + try { + double medianHouseholdIncome = Integer.parseInt(cols[5].trim()); + entries.put(zipcode, medianHouseholdIncome); + } catch (NumberFormatException e) { + + } + } + + scanner.close(); + + return ImmutableMap.copyOf(entries); + } + + private ImmutableMap<String, Long> readPopulationData(InputStream path) + throws FileNotFoundException { + Scanner scanner = new Scanner(path); + + // skip header + scanner.nextLine(); + + Map<String, Long> entries = Maps.newHashMap(); + while (scanner.hasNextLine()) { + String line = scanner.nextLine().trim(); + + if (line.length() == 0) + continue; + + String[] cols = line.split(","); + + String zipcode = cols[0].trim(); + Long population = Long.parseLong(cols[1].trim()); + + if (entries.containsKey(zipcode)) { + entries.put(zipcode, Math.max(entries.get(zipcode), population)); + } else { + entries.put(zipcode, population); + } + } + + scanner.close(); + + return ImmutableMap.copyOf(entries); + } + + private ImmutableMap<String, ZipcodeLocationRecord> readCoordinates( + InputStream path) throws FileNotFoundException { + Scanner scanner = new Scanner(path); + + // skip header + scanner.nextLine(); + + Map<String, ZipcodeLocationRecord> entries = Maps.newHashMap(); + while (scanner.hasNextLine()) { + String line = scanner.nextLine().trim(); + + String[] cols = line.split(", "); + + // remove quote marks + String zipcode = cols[0].substring(1, cols[0].length() - 1); + String state = cols[1].substring(1, cols[1].length() - 1); + Double latitude = Double + .parseDouble(cols[2].substring(1, cols[2].length() - 1)); + Double longitude = Double + .parseDouble(cols[3].substring(1, cols[3].length() - 1)); + String city = cols[4].substring(1, cols[4].length() - 1); + + Pair<Double, Double> coords = Pair.of(latitude, longitude); + + ZipcodeLocationRecord record = new ZipcodeLocationRecord(coords, city, + state); + + entries.put(zipcode, record); + } + + scanner.close(); + + return ImmutableMap.copyOf(entries); + } + + public ImmutableList<Location> readData() throws FileNotFoundException { + + ImmutableMap<String, Double> incomes = readIncomeData( + getResource(LocationConstants.INCOMES_FILE)); + ImmutableMap<String, Long> populations = readPopulationData( + getResource(LocationConstants.POPULATION_FILE)); + ImmutableMap<String, ZipcodeLocationRecord> coordinates = readCoordinates( + getResource(LocationConstants.COORDINATES_FILE)); + + Set<String> zipcodeSubset = new HashSet<String>(incomes.keySet()); + zipcodeSubset.retainAll(populations.keySet()); + zipcodeSubset.retainAll(coordinates.keySet()); + + List<Location> table = new Vector<Location>(); + for (String zipcode : zipcodeSubset) { + Location record = new Location(zipcode, + coordinates.get(zipcode).coordinates, + coordinates.get(zipcode).city, coordinates.get(zipcode).state, + incomes.get(zipcode), populations.get(zipcode)); + table.add(record); + } + + return ImmutableList.copyOf(table); + } } http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-name-generator/README.md ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-name-generator/README.md b/bigtop-data-generators/bigtop-name-generator/README.md index aa204b2..93e23a5 100644 --- a/bigtop-data-generators/bigtop-name-generator/README.md +++ b/bigtop-data-generators/bigtop-name-generator/README.md @@ -24,28 +24,4 @@ their frequencies according to the U.S. Census. Building and Testing -------------------- -We use the Gradle build system for the BPS data generator so you'll need -to install Gradle on your system. -Once that's done, you can use gradle to run the included unit tests -and build the data generator jar. - -To build: - - $ gradle build - -This will create several directories and a jar located at: - - build/libs/bigtop-name-generator-1.1.0-SNAPSHOT.jar - -Building automatically runs the included unit tests. If you would prefer -to just run the unit tests, you can do so by: - - $ gradle test - -To clean up the build files, run: - - $ gradle clean - -To install a jar into your local maven repository: - - $ gradle install +This project is part of a Gradle multiproject build. Please see the README in the parent directory for build instructions. http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-name-generator/build.gradle ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-name-generator/build.gradle b/bigtop-data-generators/bigtop-name-generator/build.gradle index 0bff824..133dc93 100644 --- a/bigtop-data-generators/bigtop-name-generator/build.gradle +++ b/bigtop-data-generators/bigtop-name-generator/build.gradle @@ -13,51 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -apply plugin: 'eclipse' -apply plugin: 'groovy' -apply plugin: 'java' -apply plugin: 'maven' -group = 'org.apache.bigtop' -version = '1.1.0-SNAPSHOT' jar { - - from { - configurations.runtime.collect { - it.isDirectory() ? it : zipTree(it) - } - } - - manifest { - attributes 'Title': 'BigTop Name Generator', 'Version': version + from { + configurations.runtime.collect { + it.isDirectory() ? it : zipTree(it) } -} - -repositories { - mavenLocal() - mavenCentral() -} - -test { - // show standard out and error on console - testLogging.showStandardStreams = true - - // listen to events in the test execution lifecycle - beforeTest { descriptor -> - logger.lifecycle("Running test: " + descriptor) - } - - // listen to standard out and standard error of the test JVM(s) - onOutput { descriptor, event -> - logger.lifecycle("Test: " + descriptor + " produced standard out/err: " + event.message ) - } + } + manifest { + attributes 'Title': 'BigTop Name Generator', 'Version': version + } } dependencies { - compile 'com.google.guava:guava:18.0' - compile 'org.apache.commons:commons-lang3:3.4' - compile project(":bigtop-samplers") + compile 'com.google.guava:guava:18.0' + compile 'org.apache.commons:commons-lang3:3.4' + compile project(":bigtop-samplers") - testCompile 'junit:junit:4.+' + testCompile 'junit:junit:4.+' } http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-samplers/README.md ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-samplers/README.md b/bigtop-data-generators/bigtop-samplers/README.md index 3a91c1b..5fb4a55 100644 --- a/bigtop-data-generators/bigtop-samplers/README.md +++ b/bigtop-data-generators/bigtop-samplers/README.md @@ -22,28 +22,4 @@ functions, probability mass functions, weight functions and samplers. Building and Testing -------------------- -We use the Gradle build system for the BPS data generator so you'll need -to install Gradle on your system. -Once that's done, you can use gradle to run the included unit tests -and build the data generator jar. - -To build: - - $ gradle build - -This will create several directories and a jar located at: - - build/libs/bigtop-samplers-0.9.0-SNAPSHOT.jar - -Building automatically runs the included unit tests. If you would prefer -to just run the unit tests, you can do so by: - - $ gradle test - -To clean up the build files, run: - - $ gradle clean - -To install a jar into your local maven repository: - - $ gradle install +This project is part of a multiproject Gradle build. Please see the README in the parent directory for instructions on building. http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-samplers/build.gradle ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-samplers/build.gradle b/bigtop-data-generators/bigtop-samplers/build.gradle index 84b5fe0..1d9ced0 100644 --- a/bigtop-data-generators/bigtop-samplers/build.gradle +++ b/bigtop-data-generators/bigtop-samplers/build.gradle @@ -13,52 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -apply plugin: 'eclipse' -apply plugin: 'groovy' -apply plugin: 'java' -apply plugin: 'maven' -group = 'org.apache.bigtop' -version = '1.1.0-SNAPSHOT' jar { - - from { - configurations.runtime.collect { - it.isDirectory() ? it : zipTree(it) - } - } - - manifest { - attributes 'Title': 'BigTop Samplers', 'Version': version + from { + configurations.runtime.collect { + it.isDirectory() ? it : zipTree(it) } -} - -repositories { - mavenLocal() - mavenCentral() -} - -test { - // show standard out and error on console - testLogging.showStandardStreams = true - - // listen to events in the test execution lifecycle - beforeTest { descriptor -> - logger.lifecycle("Running test: " + descriptor) - } - - // listen to standard out and standard error of the test JVM(s) - onOutput { descriptor, event -> - logger.lifecycle("Test: " + descriptor + " produced standard out/err: " + event.message ) - } + } + manifest { + attributes 'Title': 'BigTop Samplers', 'Version': version + } } dependencies { - compile 'com.google.guava:guava:18.0' - - compile 'org.apache.commons:commons-lang3:3.4' - compile 'org.apache.commons:commons-math3:3.5' + compile 'com.google.guava:guava:18.0' + compile 'org.apache.commons:commons-lang3:3.4' + compile 'org.apache.commons:commons-math3:3.5' - testCompile 'junit:junit:4.+' + testCompile 'junit:junit:4.+' } http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-samplers/src/main/java/org/apache/bigtop/datagenerators/samplers/samplers/GammaSampler.java ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-samplers/src/main/java/org/apache/bigtop/datagenerators/samplers/samplers/GammaSampler.java b/bigtop-data-generators/bigtop-samplers/src/main/java/org/apache/bigtop/datagenerators/samplers/samplers/GammaSampler.java new file mode 100644 index 0000000..fb08b48 --- /dev/null +++ b/bigtop-data-generators/bigtop-samplers/src/main/java/org/apache/bigtop/datagenerators/samplers/samplers/GammaSampler.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bigtop.datagenerators.samplers.samplers; + +import org.apache.bigtop.datagenerators.samplers.SeedFactory; +import org.apache.commons.math3.distribution.GammaDistribution; + +public class GammaSampler implements Sampler<Double> { + private final GammaDistribution gamma; + + public GammaSampler(double k, double theta, SeedFactory seedFactory) { + gamma = new GammaDistribution(k, theta); + gamma.reseedRandomGenerator(seedFactory.getNextSeed()); + } + + @Override + public Double sample() throws Exception { + return gamma.sample(); + } + +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/README.md ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-weatherman/README.md b/bigtop-data-generators/bigtop-weatherman/README.md new file mode 100644 index 0000000..608496c --- /dev/null +++ b/bigtop-data-generators/bigtop-weatherman/README.md @@ -0,0 +1,36 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +BigTop Weatherman +================= + +Library for generating weather on a zipcode-by-zipcode basis. + +Building and Testing +-------------------- +This project is part of a multiproject Gradle build. Please see directions in the parent directory for build instructions. + +Running +------- +BigTop Weatherman is primarily designed for consumption as a library but it also provides a CLI interface in the jar. The jar can be executed as follows: + + $ java -jar build/libs/bigtop-weatherman-1.1.0-SNAPSHOT.jar outputDir zipcode simulationLength startDate seed + +For example, simulating the weather in South Bend, IN 46617: + + $ java -jar build/libs/bigtop-weatherman-1.1.0-SNAPSHOT.jar output/ 46617 365 2014-04-05 1234 + +will produce a file `output/46617.txt` containing simulated daily temperature, wind chill, wind speed, total precipitation, rainfall, and snowfall readings. http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/build.gradle ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-weatherman/build.gradle b/bigtop-data-generators/bigtop-weatherman/build.gradle new file mode 100644 index 0000000..a2835f7 --- /dev/null +++ b/bigtop-data-generators/bigtop-weatherman/build.gradle @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +jar { + from { + configurations.runtime.collect { + it.isDirectory() ? it : zipTree(it) + } + } + + manifest { + attributes 'Title': 'BigTop Weatherman', 'Version': version + attributes 'Main-Class': 'org.apache.bigtop.datagenerators.weatherman.internal.Driver' + } +} + +dependencies { + compile 'com.google.guava:guava:18.0' + compile 'org.apache.commons:commons-lang3:3.4' + compile 'joda-time:joda-time:2.8.2' + compile project(':bigtop-samplers') + compile project(':bigtop-location-data') + + testCompile 'junit:junit:4.+' +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/settings.gradle ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-weatherman/settings.gradle b/bigtop-data-generators/bigtop-weatherman/settings.gradle new file mode 100644 index 0000000..dbc292c --- /dev/null +++ b/bigtop-data-generators/bigtop-weatherman/settings.gradle @@ -0,0 +1,16 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +rootProject.name = "bigtop-weatherman" http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/WeatherGenerator.java ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/WeatherGenerator.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/WeatherGenerator.java new file mode 100644 index 0000000..0601c14 --- /dev/null +++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/WeatherGenerator.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bigtop.datagenerators.weatherman; + +import java.util.List; + +import org.apache.bigtop.datagenerators.locations.Location; +import org.apache.bigtop.datagenerators.samplers.SeedFactory; +import org.apache.bigtop.datagenerators.samplers.samplers.Sampler; +import org.apache.bigtop.datagenerators.weatherman.internal.WeatherParametersReader; +import org.apache.bigtop.datagenerators.weatherman.internal.WeatherSamplerBuilder; +import org.apache.bigtop.datagenerators.weatherman.internal.WeatherStationParameters; +import org.joda.time.LocalDate; + +/** + * Generates daily weather records for a given location. + * + * @author rnowling + * + */ +public class WeatherGenerator implements Sampler<WeatherRecord> { + private final Sampler<WeatherRecord> weatherSampler; + + /** + * Initializes the generator. + * + * @param startDate + * - first day of simulation + * @param location + * - location to simulate weather for + * @param seedFactory + * - for initializing seeds + * @throws Exception + * - if weather parameter data cannot be read + */ + public WeatherGenerator(LocalDate startDate, Location location, + SeedFactory seedFactory) throws Exception { + List<WeatherStationParameters> parameters = new WeatherParametersReader() + .readParameters(); + WeatherSamplerBuilder builder = new WeatherSamplerBuilder(parameters, + location, startDate, seedFactory); + weatherSampler = builder.build(); + } + + /** + * Generates a daily WeatherRecord for consecutive days, one day per call. + * + * @return Weather data for a single day + * @throws Exception + */ + public WeatherRecord sample() throws Exception { + return weatherSampler.sample(); + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/WeatherRecord.java ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/WeatherRecord.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/WeatherRecord.java new file mode 100644 index 0000000..04d2eaa --- /dev/null +++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/WeatherRecord.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bigtop.datagenerators.weatherman; + +import org.joda.time.LocalDate; + +/** + * Weather data for a single day and location + * + * @author rnowling + * + */ +public interface WeatherRecord { + + /** + * Day + * + * @return date + */ + public abstract LocalDate getDate(); + + /** + * Average daily temperature in units of Fahrenheit + * + * @return temperature + */ + public abstract double getTemperature(); + + /** + * Daily total precipitation measured in millimeters + * + * @return total precipitation + */ + public abstract double getPrecipitation(); + + /** + * Daily average wind speed in meters per second. + * + * @return average wind speed + */ + public abstract double getWindSpeed(); + + /** + * Daily average wind chill in Fahrenheit + * + * @return average wind chill + */ + public abstract double getWindChill(); + + /** + * Daily total rainfall in millimeters + * + * @return total rainfall + */ + public abstract double getRainFall(); + + /** + * Daily total snowfall in centimeters + * + * @return total snowfall + */ + public abstract double getSnowFall(); + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/Driver.java ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/Driver.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/Driver.java new file mode 100644 index 0000000..0bd8c41 --- /dev/null +++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/Driver.java @@ -0,0 +1,194 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bigtop.datagenerators.weatherman.internal; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.Writer; +import java.util.List; +import java.util.Random; + +import org.apache.bigtop.datagenerators.locations.Location; +import org.apache.bigtop.datagenerators.locations.LocationReader; +import org.apache.bigtop.datagenerators.samplers.SeedFactory; +import org.apache.bigtop.datagenerators.weatherman.WeatherGenerator; +import org.apache.bigtop.datagenerators.weatherman.WeatherRecord; +import org.joda.time.DateTime; +import org.joda.time.LocalDate; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import com.google.common.collect.Lists; + +public class Driver { + List<Location> locations; + Location location; + int simulationLength; + LocalDate startDate; + SeedFactory seedFactory; + File outputDir; + + static final int NPARAMS = 5; + + private void printUsage() { + String usage = "BigPetStore Data Generator\n" + "\n" + + "Usage: java -jar bigpetstore-weather-generator.java outputDir zipcode simulationLength startDate [seed]\n" + + "\n" + "outputDir - (string) directory to write files\n" + + "zipcode - (string) location zipcode\n" + + "simulationLength - (int) number of days to simulate\n" + + "startDate - (string) simulation start date in YYYY-MM-DD format\n" + + "seed - (long) seed for RNG. If not given, one is randomly generated.\n"; + + System.out.println(usage); + } + + private Location findLocation(String zipcode) { + for (Location location : locations) { + if (location.getZipcode().equals(zipcode)) + return location; + } + + return null; + } + + /** + * Returns local date from string in YYYY-MM-DD format + * + * @param text + * @return + */ + private LocalDate parseDate(String text) { + DateTimeFormatter dtFormatter = DateTimeFormat.forPattern("yyyy-MM-dd"); + DateTime dt = dtFormatter.parseDateTime(text); + + return dt.toLocalDate(); + } + + public void parseArgs(String[] args) { + if (args.length != NPARAMS && args.length != (NPARAMS - 1)) { + printUsage(); + System.exit(1); + } + + int i = -1; + + outputDir = new File(args[++i]); + if (!outputDir.exists()) { + System.err.println("Given path (" + args[i] + ") does not exist.\n"); + printUsage(); + System.exit(1); + } + + if (!outputDir.isDirectory()) { + System.err.println("Given path (" + args[i] + ") is not a directory.\n"); + printUsage(); + System.exit(1); + } + + location = findLocation(args[++i]); + if (location == null) { + System.err.println( + "No location found for given zipcode \"" + args[i] + "\""); + printUsage(); + System.exit(1); + } + + try { + simulationLength = Integer.parseInt(args[++i]); + } catch (Exception e) { + System.err.println("Unable to parse '" + args[i] + + "' as an int for simulationLength.\n"); + printUsage(); + System.exit(1); + } + + try { + startDate = parseDate(args[++i]); + } catch (Exception e) { + System.err.println("Unable to parse '" + args[i] + + "'. Expected string in 'YYYY-MM-DD' format.\n"); + printUsage(); + System.exit(1); + } + + long seed = (new Random()).nextLong(); + if (args.length == NPARAMS) { + try { + seed = Long.parseLong(args[++i]); + } catch (Exception e) { + System.err.println( + "Unable to parse '" + args[i] + "' as a long for the seed.\n"); + printUsage(); + System.exit(1); + } + } + + seedFactory = new SeedFactory(seed); + } + + private void writeWeather(Location location, List<WeatherRecord> trajectory) + throws Exception { + File outputFile = new File(outputDir.toString() + File.separator + + location.getZipcode() + ".txt"); + Writer output = new BufferedWriter(new FileWriter(outputFile)); + + output.write( + "date,city,state,zipcode,temperature,windchill,windspeed,precipitation,rainfall,snowfall\n"); + for (WeatherRecord weather : trajectory) { + String record = weather.getDate() + ","; + record += location.getCity() + ","; + record += location.getState() + ","; + record += location.getZipcode() + ","; + record += weather.getTemperature() + ","; + record += weather.getWindChill() + ","; + record += weather.getWindSpeed() + ","; + record += weather.getPrecipitation() + ","; + record += weather.getRainFall() + ","; + record += weather.getSnowFall() + "\n"; + + output.write(record); + } + + output.close(); + } + + public void run(String[] args) throws Exception { + locations = new LocationReader().readData(); + + parseArgs(args); + + WeatherGenerator generator = new WeatherGenerator(startDate, location, + seedFactory); + + LocalDate date = startDate; + LocalDate endDate = startDate.plusDays(simulationLength); + List<WeatherRecord> trajectory = Lists.newArrayList(); + + while (date.isBefore(endDate)) { + WeatherRecord weather = generator.sample(); + trajectory.add(weather); + date = weather.getDate(); + } + + writeWeather(location, trajectory); + } + + public static void main(String[] args) throws Exception { + Driver driver = new Driver(); + driver.run(args); + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/PrecipitationSampler.java ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/PrecipitationSampler.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/PrecipitationSampler.java new file mode 100644 index 0000000..118e245 --- /dev/null +++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/PrecipitationSampler.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bigtop.datagenerators.weatherman.internal; + +import org.apache.bigtop.datagenerators.samplers.SeedFactory; +import org.apache.bigtop.datagenerators.samplers.samplers.ConditionalSampler; +import org.apache.bigtop.datagenerators.samplers.samplers.ExponentialSampler; +import org.apache.bigtop.datagenerators.samplers.samplers.Sampler; + +public class PrecipitationSampler implements + ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> { + private final Sampler<Double> precipitationSampler; + + public PrecipitationSampler(double averagePrecipitation, + SeedFactory seedFactory) { + precipitationSampler = new ExponentialSampler(1.0 / averagePrecipitation, + seedFactory); + } + + public WeatherRecordBuilder sample(WeatherRecordBuilder record) + throws Exception { + double temp = record.getTemperature(); + double precipitation = precipitationSampler.sample(); + record.setPrecipitation(precipitation); + + double fractionRain = 1.0 + / (1.0 + Math.exp(-WeatherConstants.PRECIPITATION_A + * (temp - WeatherConstants.PRECIPITATION_B))); + + double rainfall = fractionRain * precipitation; + double snowfall = WeatherConstants.PRECIPITATION_TO_SNOWFALL + * (1.0 - fractionRain) * precipitation; + record.setRainFall(rainfall); + record.setSnowFall(snowfall); + + return record; + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/TemperatureSampler.java ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/TemperatureSampler.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/TemperatureSampler.java new file mode 100644 index 0000000..b64b6e0 --- /dev/null +++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/TemperatureSampler.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bigtop.datagenerators.weatherman.internal; + +import org.apache.bigtop.datagenerators.samplers.SeedFactory; +import org.apache.bigtop.datagenerators.samplers.samplers.ConditionalSampler; +import org.apache.bigtop.datagenerators.samplers.samplers.GaussianSampler; +import org.apache.bigtop.datagenerators.samplers.samplers.Sampler; +import org.joda.time.LocalDate; + +public class TemperatureSampler implements + ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> { + final private Sampler<Double> R; + final private double average; + final private double coeffReal; + final private double coeffImag; + + private LocalDate date; + private double noise; + + public TemperatureSampler(LocalDate startDate, double tempAverage, + double tempRealCoeff, double tempImagCoeff, double tempSigma, + SeedFactory seedFactory) { + R = new GaussianSampler(0.0, tempSigma, seedFactory); + + this.average = tempAverage; + this.coeffReal = tempRealCoeff; + this.coeffImag = tempImagCoeff; + + date = startDate; + noise = 0.0; + } + + public WeatherRecordBuilder sample(WeatherRecordBuilder weatherRecord) + throws Exception { + double temp = 0.0; + while (date.isEqual(weatherRecord.getDate()) + || date.isBefore(weatherRecord.getDate())) { + double dayOfYear = date.getDayOfYear(); + temp = average + + coeffReal * Math.cos(-2.0 * Math.PI * dayOfYear + / WeatherConstants.TEMPERATURE_PERIOD) + + coeffImag * Math.sin(-2.0 * Math.PI * dayOfYear + / WeatherConstants.TEMPERATURE_PERIOD) + + noise; + + noise += -1.0 * noise * WeatherConstants.TEMPERATURE_GAMMA + * WeatherConstants.WEATHER_TIMESTEP + + Math.sqrt(WeatherConstants.WEATHER_TIMESTEP) * R.sample(); + + date = date.plusDays(WeatherConstants.WEATHER_TIMESTEP); + } + + weatherRecord.setTemperature(temp); + + return weatherRecord; + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherConstants.java ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherConstants.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherConstants.java new file mode 100644 index 0000000..8346160 --- /dev/null +++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherConstants.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bigtop.datagenerators.weatherman.internal; + +import java.io.File; + +public class WeatherConstants { + public static final File WEATHER_PARAMETERS_FILE = new File( + "weather_parameters.csv"); + + public static final double TEMPERATURE_GAMMA = 0.5; // 2 / day + public static final int WEATHER_TIMESTEP = 1; // days + public static final double TEMPERATURE_PERIOD = 365.0; // days + + public static final double PRECIPITATION_A = 0.2; + public static final double PRECIPITATION_B = 27.0; + public static final double PRECIPITATION_TO_SNOWFALL = 10.0; + + public static final double WIND_CHILL_PROBABILITY_A = 0.8; + public static final double WIND_CHILL_PROBABILITY_B = 0.5; + public static final double WIND_CHILL_PROBABILITY_C = 10.0; // F + public static final double WIND_CHILL_PROBABILITY_D = 0.2; + + public static final double WIND_SPEED_PROBABILITY_A = -0.5; + public static final double WIND_SPEED_PROBABILITY_B = 0.8; + public static final double WIND_SPEED_PROBABILITY_C = 17.5; // mph + public static final double WIND_SPEED_PROBABILITY_D = 1.0; + + public static final double SNOWFALL_PROBABILITY_A = -0.8; + public static final double SNOWFALL_PROBABILITY_B = 10.0; + public static final double SNOWFALL_PROBABILITY_C = 0.75; // in + public static final double SNOWFALL_PROBABILITY_D = 1.0; + + public static final double RAINFALL_PROBABILITY_A = -0.6; + public static final double RAINFALL_PROBABILITY_B = 7.5; + public static final double RAINFALL_PROBABILITY_C = 0.75; // in + public static final double RAINFALL_PROBABILITY_D = 1.0; + + public static final double M_PER_S_TO_MPH = 2.23694; +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherParametersReader.java ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherParametersReader.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherParametersReader.java new file mode 100644 index 0000000..e8097d6 --- /dev/null +++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherParametersReader.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bigtop.datagenerators.weatherman.internal; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.InputStream; +import java.util.List; +import java.util.Scanner; + +import org.apache.commons.lang3.tuple.Pair; + +import com.google.common.collect.Lists; + +public class WeatherParametersReader { + private InputStream getResource(File filename) throws Exception { + InputStream stream = getClass() + .getResourceAsStream("/input_data/" + filename); + return new BufferedInputStream(stream); + } + + private WeatherStationParameters parseLine(String line) { + line = line.trim(); + String[] cols = line.split(","); + + String WBAN = cols[0]; + String city = cols[1]; + String state = cols[2]; + double latitude = Double.parseDouble(cols[3]); + double longitude = Double.parseDouble(cols[4]); + Pair<Double, Double> coordinates = Pair.of(latitude, longitude); + double temperatureAverage = Double.parseDouble(cols[5]); + double temperatureRealCoeff = Double.parseDouble(cols[6]); + double temperatureImagCoeff = Double.parseDouble(cols[7]); + double temperatureDerivStd = Double.parseDouble(cols[8]); + double precipitationAverage = Double.parseDouble(cols[9]); + double windSpeedRealCoeff = Double.parseDouble(cols[10]); + double windSpeedImagCoeff = Double.parseDouble(cols[11]); + double windSpeedK = Double.parseDouble(cols[12]); + double windSpeedTheta = Double.parseDouble(cols[13]); + + return new WeatherStationParameters(WBAN, city, state, coordinates, + temperatureAverage, temperatureRealCoeff, temperatureImagCoeff, + temperatureDerivStd, precipitationAverage, windSpeedRealCoeff, + windSpeedImagCoeff, windSpeedK, windSpeedTheta); + } + + public List<WeatherStationParameters> readParameters() throws Exception { + InputStream inputStream = getResource( + WeatherConstants.WEATHER_PARAMETERS_FILE); + Scanner scanner = new Scanner(inputStream); + + // skip header + scanner.nextLine(); + + List<WeatherStationParameters> parameterList = Lists.newArrayList(); + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + WeatherStationParameters parameters = parseLine(line); + parameterList.add(parameters); + } + + scanner.close(); + + return parameterList; + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherRecordBuilder.java ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherRecordBuilder.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherRecordBuilder.java new file mode 100644 index 0000000..393414e --- /dev/null +++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherRecordBuilder.java @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bigtop.datagenerators.weatherman.internal; + +import java.io.Serializable; + +import org.apache.bigtop.datagenerators.weatherman.WeatherRecord; +import org.joda.time.LocalDate; + +public class WeatherRecordBuilder implements Serializable, WeatherRecord { + private static final long serialVersionUID = -6397575341624071821L; + + private LocalDate date; + private Double temperature; + private Double precipitation; + private Double windSpeed; + private Double windChill; + private Double rainFall; + private Double snowFall; + + public WeatherRecordBuilder(LocalDate date) { + this.date = date; + } + + public WeatherRecordBuilder(LocalDate date, double temperature, + double precipitation, double windSpeed, double windChill, + double rainFall, double snowFall) { + this.date = date; + this.temperature = temperature; + this.precipitation = precipitation; + this.windSpeed = windSpeed; + this.windChill = windChill; + this.rainFall = rainFall; + this.snowFall = snowFall; + } + + /* + * (non-Javadoc) + * + * @see org.apache.bigtop.bigpetstore.datagenerator.generators.weather. + * WeatherRecord#getDate() + */ + @Override + public LocalDate getDate() { + return date; + } + + /* + * (non-Javadoc) + * + * @see org.apache.bigtop.bigpetstore.datagenerator.generators.weather. + * WeatherRecord#getTemperature() + */ + @Override + public double getTemperature() { + return temperature; + } + + /* + * (non-Javadoc) + * + * @see org.apache.bigtop.bigpetstore.datagenerator.generators.weather. + * WeatherRecord#getPrecipitation() + */ + @Override + public double getPrecipitation() { + return precipitation; + } + + /* + * (non-Javadoc) + * + * @see org.apache.bigtop.bigpetstore.datagenerator.generators.weather. + * WeatherRecord#getWindSpeed() + */ + @Override + public double getWindSpeed() { + return windSpeed; + } + + /* + * (non-Javadoc) + * + * @see org.apache.bigtop.bigpetstore.datagenerator.generators.weather. + * WeatherRecord#getWindChill() + */ + @Override + public double getWindChill() { + return windChill; + } + + /* + * (non-Javadoc) + * + * @see org.apache.bigtop.bigpetstore.datagenerator.generators.weather. + * WeatherRecord#getRainFall() + */ + @Override + public double getRainFall() { + return rainFall; + } + + /* + * (non-Javadoc) + * + * @see org.apache.bigtop.bigpetstore.datagenerator.generators.weather. + * WeatherRecord#getSnowFall() + */ + @Override + public double getSnowFall() { + return snowFall; + } + + public WeatherRecord build() { + if (temperature == null || precipitation == null || windSpeed == null + || windChill == null || rainFall == null || snowFall == null) { + throw new IllegalStateException("Not all fields have been initialized."); + } + + return this; + } + + public void setDate(LocalDate date) { + this.date = date; + } + + public void setTemperature(Double temperature) { + this.temperature = temperature; + } + + public void setPrecipitation(Double precipitation) { + this.precipitation = precipitation; + } + + public void setWindSpeed(Double windSpeed) { + this.windSpeed = windSpeed; + } + + public void setWindChill(Double windChill) { + this.windChill = windChill; + } + + public void setRainFall(Double rainFall) { + this.rainFall = rainFall; + } + + public void setSnowFall(Double snowFall) { + this.snowFall = snowFall; + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherSampler.java ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherSampler.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherSampler.java new file mode 100644 index 0000000..3885caa --- /dev/null +++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherSampler.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bigtop.datagenerators.weatherman.internal; + +import org.apache.bigtop.datagenerators.samplers.samplers.ConditionalSampler; +import org.apache.bigtop.datagenerators.samplers.samplers.Sampler; +import org.apache.bigtop.datagenerators.weatherman.WeatherRecord; +import org.joda.time.LocalDate; + +public class WeatherSampler implements Sampler<WeatherRecord> { + private final ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> tempSampler; + private final ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> windSpeedSampler; + private final ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> precipitationSampler; + private LocalDate date; + + public WeatherSampler(LocalDate startDate, + ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> tempSampler, + ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> windSpeedSampler, + ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> precipitationSampler) { + this.tempSampler = tempSampler; + this.windSpeedSampler = windSpeedSampler; + this.precipitationSampler = precipitationSampler; + date = startDate; + } + + public WeatherRecord sample() throws Exception { + WeatherRecordBuilder weatherRecord = new WeatherRecordBuilder(date); + + weatherRecord = tempSampler.sample(weatherRecord); + weatherRecord = windSpeedSampler.sample(weatherRecord); + weatherRecord = precipitationSampler.sample(weatherRecord); + + date = date.plusDays(WeatherConstants.WEATHER_TIMESTEP); + + return weatherRecord.build(); + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherSamplerBuilder.java ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherSamplerBuilder.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherSamplerBuilder.java new file mode 100644 index 0000000..40128ae --- /dev/null +++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherSamplerBuilder.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bigtop.datagenerators.weatherman.internal; + +import java.util.Collection; + +import org.apache.bigtop.datagenerators.locations.Location; +import org.apache.bigtop.datagenerators.samplers.SeedFactory; +import org.apache.bigtop.datagenerators.samplers.samplers.ConditionalSampler; +import org.apache.bigtop.datagenerators.samplers.samplers.Sampler; +import org.apache.bigtop.datagenerators.weatherman.WeatherRecord; +import org.joda.time.LocalDate; + +public class WeatherSamplerBuilder { + + private final WeatherStationParameters parameters; + private final SeedFactory seedFactory; + private final LocalDate startDate; + + public WeatherSamplerBuilder( + Collection<WeatherStationParameters> weatherParameters, + Location location, LocalDate startDate, SeedFactory seedFactory) { + parameters = findClosest(weatherParameters, location); + this.seedFactory = seedFactory; + this.startDate = startDate; + } + + private WeatherStationParameters findClosest( + Collection<WeatherStationParameters> weatherParameters, + Location location) { + WeatherStationParameters closestStation = null; + double minDist = Double.MAX_VALUE; + + for (WeatherStationParameters parameters : weatherParameters) { + double dist = location.distance(parameters.getCoordinates()); + if (dist < minDist) { + minDist = dist; + closestStation = parameters; + } + } + + return closestStation; + } + + private ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> buildTempSampler() { + return new TemperatureSampler(startDate, parameters.getTemperatureAverage(), + parameters.getTemperatureRealCoeff(), + parameters.getTemperatureImagCoeff(), + parameters.getTemperatureDerivStd(), seedFactory); + } + + private ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> buildWindSpeedSampler() { + return new WindSpeedSampler(parameters.getWindSpeedRealCoeff(), + parameters.getWindSpeedImagCoeff(), parameters.getWindSpeedK(), + parameters.getWindSpeedTheta(), seedFactory); + } + + private ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> buildPrecipitationSampler() { + return new PrecipitationSampler(parameters.getPrecipitationAverage(), + seedFactory); + } + + public Sampler<WeatherRecord> build() { + return new WeatherSampler(startDate, buildTempSampler(), + buildWindSpeedSampler(), buildPrecipitationSampler()); + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherStationParameters.java ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherStationParameters.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherStationParameters.java new file mode 100644 index 0000000..7edba34 --- /dev/null +++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WeatherStationParameters.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bigtop.datagenerators.weatherman.internal; + +import java.io.Serializable; + +import org.apache.commons.lang3.tuple.Pair; + +public class WeatherStationParameters implements Serializable { + private static final long serialVersionUID = -7268791467819627718L; + + private String WBAN; + private String city; + private String state; + private Pair<Double, Double> coordinates; + private double temperatureAverage; + private double temperatureRealCoeff; + private double temperatureImagCoeff; + private double temperatureDerivStd; + private double precipitationAverage; + private double windSpeedRealCoeff; + private double windSpeedImagCoeff; + private double windSpeedK; + private double windSpeedTheta; + + public WeatherStationParameters(String WBAN, String city, String state, + Pair<Double, Double> coordinates, double temperatureAverage, + double temperatureRealCoeff, double temperatureImagCoeff, + double temperatureDerivStd, double precipitationAverage, + double windSpeedRealCoeff, double windSpeedImagCoeff, + double windSpeedK, double windSpeedTheta) { + this.city = city; + this.state = state; + this.coordinates = coordinates; + this.temperatureAverage = temperatureAverage; + this.temperatureRealCoeff = temperatureRealCoeff; + this.temperatureImagCoeff = temperatureImagCoeff; + this.temperatureDerivStd = temperatureDerivStd; + this.precipitationAverage = precipitationAverage; + this.windSpeedRealCoeff = windSpeedRealCoeff; + this.windSpeedImagCoeff = windSpeedImagCoeff; + this.windSpeedK = windSpeedK; + this.windSpeedTheta = windSpeedTheta; + } + + public String getWBAN() { + return WBAN; + } + + public String getCity() { + return city; + } + + public String getState() { + return state; + } + + public Pair<Double, Double> getCoordinates() { + return coordinates; + } + + public double getTemperatureAverage() { + return temperatureAverage; + } + + public double getTemperatureRealCoeff() { + return temperatureRealCoeff; + } + + public double getTemperatureImagCoeff() { + return temperatureImagCoeff; + } + + public double getTemperatureDerivStd() { + return temperatureDerivStd; + } + + public double getPrecipitationAverage() { + return precipitationAverage; + } + + public double getWindSpeedRealCoeff() { + return windSpeedRealCoeff; + } + + public double getWindSpeedImagCoeff() { + return windSpeedImagCoeff; + } + + public double getWindSpeedK() { + return windSpeedK; + } + + public double getWindSpeedTheta() { + return windSpeedTheta; + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad88e741/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WindSpeedSampler.java ---------------------------------------------------------------------- diff --git a/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WindSpeedSampler.java b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WindSpeedSampler.java new file mode 100644 index 0000000..c445432 --- /dev/null +++ b/bigtop-data-generators/bigtop-weatherman/src/main/java/org/apache/bigtop/datagenerators/weatherman/internal/WindSpeedSampler.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bigtop.datagenerators.weatherman.internal; + +import org.apache.bigtop.datagenerators.samplers.SeedFactory; +import org.apache.bigtop.datagenerators.samplers.samplers.ConditionalSampler; +import org.apache.bigtop.datagenerators.samplers.samplers.GammaSampler; +import org.apache.bigtop.datagenerators.samplers.samplers.Sampler; + +public class WindSpeedSampler implements + ConditionalSampler<WeatherRecordBuilder, WeatherRecordBuilder> { + private final double coeffReal; + private final double coeffImag; + private final Sampler<Double> gamma; + + public WindSpeedSampler(double windSpeedRealCoeff, double windSpeedImagCoeff, + double windSpeedK, double windSpeedTheta, SeedFactory seedFactory) { + coeffReal = windSpeedRealCoeff; + coeffImag = windSpeedImagCoeff; + + gamma = new GammaSampler(windSpeedK, windSpeedTheta, seedFactory); + + } + + /** + * + * @param temp + * - given in Fahrenheit + * @param windSpeed + * - given in MPH + * @return + */ + private Double windChill(double temp, double windSpeed) { + double v_16 = Math.pow(windSpeed, 0.16); + double windChill = 35.74 + 0.6215 * temp - 35.74 * v_16 + + 0.4275 * temp * v_16; + + return windChill; + } + + public WeatherRecordBuilder sample(WeatherRecordBuilder weatherRecord) + throws Exception { + + double dayOfYear = weatherRecord.getDate().getDayOfYear(); + // meters/second + double windSpeed = Math.max(0.0, + coeffReal + * Math.cos(-2.0 * Math.PI * dayOfYear + / WeatherConstants.TEMPERATURE_PERIOD) + + coeffImag * Math.sin(-2.0 * Math.PI * dayOfYear + / WeatherConstants.TEMPERATURE_PERIOD) + gamma.sample()); + + double windChill = windChill(weatherRecord.getTemperature(), + windSpeed * WeatherConstants.M_PER_S_TO_MPH); + + weatherRecord.setWindSpeed(windSpeed); + weatherRecord.setWindChill(windChill); + + return weatherRecord; + } +}
