- start working on examples showing how to deploy ALS model.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/6f9d8527 Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/6f9d8527 Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/6f9d8527 Branch: refs/heads/master Commit: 6f9d8527bb79e80fe00164f40bf045df5b684b1e Parents: 319ae49 Author: DO YUNG YOON <[email protected]> Authored: Wed May 2 15:38:32 2018 +0900 Committer: DO YUNG YOON <[email protected]> Committed: Wed May 2 15:38:32 2018 +0900 ---------------------------------------------------------------------- project/Common.scala | 4 + .../s2graph/core/model/AnnoyModelFetcher.scala | 17 +- s2core/src/test/resources/item_factors.ann | Bin 0 -> 137800 bytes s2core/src/test/resources/movie.dict | 1682 ++++++++++++++++++ s2core/src/test/resources/user_factors.ann | Bin 0 -> 76804 bytes .../apache/s2graph/core/model/FetcherTest.scala | 2 +- s2jobs/build.sbt | 5 +- .../custom/process/ALSAnnoyBuildProcess.scala | 87 + .../process/ALSAnnoyBuildProcessTest.scala | 104 ++ 9 files changed, 1893 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6f9d8527/project/Common.scala ---------------------------------------------------------------------- diff --git a/project/Common.scala b/project/Common.scala index 96109d3..04279f4 100644 --- a/project/Common.scala +++ b/project/Common.scala @@ -33,6 +33,10 @@ object Common { val KafkaVersion = "0.10.2.1" + val annoy4sVersion = "0.6.0" + + val tensorflowVersion = "1.7.0" + /** use Log4j 1.2.17 as the SLF4j backend in runtime, with bridging libraries to forward JCL and JUL logs to SLF4j */ val loggingRuntime = Seq( "log4j" % "log4j" % "1.2.17", http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6f9d8527/s2core/src/main/scala/org/apache/s2graph/core/model/AnnoyModelFetcher.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/model/AnnoyModelFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/model/AnnoyModelFetcher.scala index df083fa..5e0b979 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/model/AnnoyModelFetcher.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/model/AnnoyModelFetcher.scala @@ -17,10 +17,15 @@ object AnnoyModelFetcher { val DimensionKey = "annoyIndexDimension" val IndexTypeKey = "annoyIndexType" - def loadDictFromLocal(file: File): Array[String] = { - Source.fromFile(file).getLines().map { line => - line.stripMargin - }.toArray + def loadDictFromLocal(file: File): Map[Int, String] = { + Source.fromFile(file).getLines().zipWithIndex.map { case (line, _idx) => + val tokens = line.stripMargin.split("\t") + if (tokens.length < 2) { + (_idx + 1, tokens.head) + } else { + (tokens.head.toInt, tokens.tail.head) + } + }.toMap } def buildIndex(config: Config): ANNIndexWithDict = { @@ -36,8 +41,8 @@ object AnnoyModelFetcher { } } -case class ANNIndexWithDict(index: ANNIndex, dict: Array[String]) { - val dictRev = dict.zipWithIndex.toMap +case class ANNIndexWithDict(index: ANNIndex, dict: Map[Int, String]) { + val dictRev = dict.map(kv => kv._2 -> kv._1) } class AnnoyModelFetcher(val graph: S2GraphLike) extends Fetcher { http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6f9d8527/s2core/src/test/resources/item_factors.ann ---------------------------------------------------------------------- diff --git a/s2core/src/test/resources/item_factors.ann b/s2core/src/test/resources/item_factors.ann new file mode 100644 index 0000000..190485c Binary files /dev/null and b/s2core/src/test/resources/item_factors.ann differ http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6f9d8527/s2core/src/test/resources/movie.dict ---------------------------------------------------------------------- diff --git a/s2core/src/test/resources/movie.dict b/s2core/src/test/resources/movie.dict new file mode 100644 index 0000000..62a0f5d --- /dev/null +++ b/s2core/src/test/resources/movie.dict @@ -0,0 +1,1682 @@ +1 Toy Story (1995) +2 GoldenEye (1995) +3 Four Rooms (1995) +4 Get Shorty (1995) +5 Copycat (1995) +6 Shanghai Triad (Yao a yao yao dao waipo qiao) (1995) +7 Twelve Monkeys (1995) +8 Babe (1995) +9 Dead Man Walking (1995) +10 Richard III (1995) +11 Seven (Se7en) (1995) +12 Usual Suspects, The (1995) +13 Mighty Aphrodite (1995) +14 Postino, Il (1994) +15 Mr. Holland's Opus (1995) +16 French Twist (Gazon maudit) (1995) +17 From Dusk Till Dawn (1996) +18 White Balloon, The (1995) +19 Antonia's Line (1995) +20 Angels and Insects (1995) +21 Muppet Treasure Island (1996) +22 Braveheart (1995) +23 Taxi Driver (1976) +24 Rumble in the Bronx (1995) +25 Birdcage, The (1996) +26 Brothers McMullen, The (1995) +27 Bad Boys (1995) +28 Apollo 13 (1995) +29 Batman Forever (1995) +30 Belle de jour (1967) +31 Crimson Tide (1995) +32 Crumb (1994) +33 Desperado (1995) +34 Doom Generation, The (1995) +35 Free Willy 2: The Adventure Home (1995) +36 Mad Love (1995) +37 Nadja (1994) +38 Net, The (1995) +39 Strange Days (1995) +40 To Wong Foo, Thanks for Everything! Julie Newmar (1995) +41 Billy Madison (1995) +42 Clerks (1994) +43 Disclosure (1994) +44 Dolores Claiborne (1994) +45 Eat Drink Man Woman (1994) +46 Exotica (1994) +47 Ed Wood (1994) +48 Hoop Dreams (1994) +49 I.Q. (1994) +50 Star Wars (1977) +51 Legends of the Fall (1994) +52 Madness of King George, The (1994) +53 Natural Born Killers (1994) +54 Outbreak (1995) +55 Professional, The (1994) +56 Pulp Fiction (1994) +57 Priest (1994) +58 Quiz Show (1994) +59 Three Colors: Red (1994) +60 Three Colors: Blue (1993) +61 Three Colors: White (1994) +62 Stargate (1994) +63 Santa Clause, The (1994) +64 Shawshank Redemption, The (1994) +65 What's Eating Gilbert Grape (1993) +66 While You Were Sleeping (1995) +67 Ace Ventura: Pet Detective (1994) +68 Crow, The (1994) +69 Forrest Gump (1994) +70 Four Weddings and a Funeral (1994) +71 Lion King, The (1994) +72 Mask, The (1994) +73 Maverick (1994) +74 Faster Pussycat! Kill! Kill! (1965) +75 Brother Minister: The Assassination of Malcolm X (1994) +76 Carlito's Way (1993) +77 Firm, The (1993) +78 Free Willy (1993) +79 Fugitive, The (1993) +80 Hot Shots! Part Deux (1993) +81 Hudsucker Proxy, The (1994) +82 Jurassic Park (1993) +83 Much Ado About Nothing (1993) +84 Robert A. Heinlein's The Puppet Masters (1994) +85 Ref, The (1994) +86 Remains of the Day, The (1993) +87 Searching for Bobby Fischer (1993) +88 Sleepless in Seattle (1993) +89 Blade Runner (1982) +90 So I Married an Axe Murderer (1993) +91 Nightmare Before Christmas, The (1993) +92 True Romance (1993) +93 Welcome to the Dollhouse (1995) +94 Home Alone (1990) +95 Aladdin (1992) +96 Terminator 2: Judgment Day (1991) +97 Dances with Wolves (1990) +98 Silence of the Lambs, The (1991) +99 Snow White and the Seven Dwarfs (1937) +100 Fargo (1996) +101 Heavy Metal (1981) +102 Aristocats, The (1970) +103 All Dogs Go to Heaven 2 (1996) +104 Theodore Rex (1995) +105 Sgt. Bilko (1996) +106 Diabolique (1996) +107 Moll Flanders (1996) +108 Kids in the Hall: Brain Candy (1996) +109 Mystery Science Theater 3000: The Movie (1996) +110 Operation Dumbo Drop (1995) +111 Truth About Cats & Dogs, The (1996) +112 Flipper (1996) +113 Horseman on the Roof, The (Hussard sur le toit, Le) (1995) +114 Wallace & Gromit: The Best of Aardman Animation (1996) +115 Haunted World of Edward D. Wood Jr., The (1995) +116 Cold Comfort Farm (1995) +117 Rock, The (1996) +118 Twister (1996) +119 Maya Lin: A Strong Clear Vision (1994) +120 Striptease (1996) +121 Independence Day (ID4) (1996) +122 Cable Guy, The (1996) +123 Frighteners, The (1996) +124 Lone Star (1996) +125 Phenomenon (1996) +126 Spitfire Grill, The (1996) +127 Godfather, The (1972) +128 Supercop (1992) +129 Bound (1996) +130 Kansas City (1996) +131 Breakfast at Tiffany's (1961) +132 Wizard of Oz, The (1939) +133 Gone with the Wind (1939) +134 Citizen Kane (1941) +135 2001: A Space Odyssey (1968) +136 Mr. Smith Goes to Washington (1939) +137 Big Night (1996) +138 D3: The Mighty Ducks (1996) +139 Love Bug, The (1969) +140 Homeward Bound: The Incredible Journey (1993) +141 20,000 Leagues Under the Sea (1954) +142 Bedknobs and Broomsticks (1971) +143 Sound of Music, The (1965) +144 Die Hard (1988) +145 Lawnmower Man, The (1992) +146 Unhook the Stars (1996) +147 Long Kiss Goodnight, The (1996) +148 Ghost and the Darkness, The (1996) +149 Jude (1996) +150 Swingers (1996) +151 Willy Wonka and the Chocolate Factory (1971) +152 Sleeper (1973) +153 Fish Called Wanda, A (1988) +154 Monty Python's Life of Brian (1979) +155 Dirty Dancing (1987) +156 Reservoir Dogs (1992) +157 Platoon (1986) +158 Weekend at Bernie's (1989) +159 Basic Instinct (1992) +160 Glengarry Glen Ross (1992) +161 Top Gun (1986) +162 On Golden Pond (1981) +163 Return of the Pink Panther, The (1974) +164 Abyss, The (1989) +165 Jean de Florette (1986) +166 Manon of the Spring (Manon des sources) (1986) +167 Private Benjamin (1980) +168 Monty Python and the Holy Grail (1974) +169 Wrong Trousers, The (1993) +170 Cinema Paradiso (1988) +171 Delicatessen (1991) +172 Empire Strikes Back, The (1980) +173 Princess Bride, The (1987) +174 Raiders of the Lost Ark (1981) +175 Brazil (1985) +176 Aliens (1986) +177 Good, The Bad and The Ugly, The (1966) +178 12 Angry Men (1957) +179 Clockwork Orange, A (1971) +180 Apocalypse Now (1979) +181 Return of the Jedi (1983) +182 GoodFellas (1990) +183 Alien (1979) +184 Army of Darkness (1993) +185 Psycho (1960) +186 Blues Brothers, The (1980) +187 Godfather: Part II, The (1974) +188 Full Metal Jacket (1987) +189 Grand Day Out, A (1992) +190 Henry V (1989) +191 Amadeus (1984) +192 Raging Bull (1980) +193 Right Stuff, The (1983) +194 Sting, The (1973) +195 Terminator, The (1984) +196 Dead Poets Society (1989) +197 Graduate, The (1967) +198 Nikita (La Femme Nikita) (1990) +199 Bridge on the River Kwai, The (1957) +200 Shining, The (1980) +201 Evil Dead II (1987) +202 Groundhog Day (1993) +203 Unforgiven (1992) +204 Back to the Future (1985) +205 Patton (1970) +206 Akira (1988) +207 Cyrano de Bergerac (1990) +208 Young Frankenstein (1974) +209 This Is Spinal Tap (1984) +210 Indiana Jones and the Last Crusade (1989) +211 M*A*S*H (1970) +212 Unbearable Lightness of Being, The (1988) +213 Room with a View, A (1986) +214 Pink Floyd - The Wall (1982) +215 Field of Dreams (1989) +216 When Harry Met Sally... (1989) +217 Bram Stoker's Dracula (1992) +218 Cape Fear (1991) +219 Nightmare on Elm Street, A (1984) +220 Mirror Has Two Faces, The (1996) +221 Breaking the Waves (1996) +222 Star Trek: First Contact (1996) +223 Sling Blade (1996) +224 Ridicule (1996) +225 101 Dalmatians (1996) +226 Die Hard 2 (1990) +227 Star Trek VI: The Undiscovered Country (1991) +228 Star Trek: The Wrath of Khan (1982) +229 Star Trek III: The Search for Spock (1984) +230 Star Trek IV: The Voyage Home (1986) +231 Batman Returns (1992) +232 Young Guns (1988) +233 Under Siege (1992) +234 Jaws (1975) +235 Mars Attacks! (1996) +236 Citizen Ruth (1996) +237 Jerry Maguire (1996) +238 Raising Arizona (1987) +239 Sneakers (1992) +240 Beavis and Butt-head Do America (1996) +241 Last of the Mohicans, The (1992) +242 Kolya (1996) +243 Jungle2Jungle (1997) +244 Smilla's Sense of Snow (1997) +245 Devil's Own, The (1997) +246 Chasing Amy (1997) +247 Turbo: A Power Rangers Movie (1997) +248 Grosse Pointe Blank (1997) +249 Austin Powers: International Man of Mystery (1997) +250 Fifth Element, The (1997) +251 Shall We Dance? (1996) +252 Lost World: Jurassic Park, The (1997) +253 Pillow Book, The (1995) +254 Batman & Robin (1997) +255 My Best Friend's Wedding (1997) +256 When the Cats Away (Chacun cherche son chat) (1996) +257 Men in Black (1997) +258 Contact (1997) +259 George of the Jungle (1997) +260 Event Horizon (1997) +261 Air Bud (1997) +262 In the Company of Men (1997) +263 Steel (1997) +264 Mimic (1997) +265 Hunt for Red October, The (1990) +266 Kull the Conqueror (1997) +267 unknown +268 Chasing Amy (1997) +269 Full Monty, The (1997) +270 Gattaca (1997) +271 Starship Troopers (1997) +272 Good Will Hunting (1997) +273 Heat (1995) +274 Sabrina (1995) +275 Sense and Sensibility (1995) +276 Leaving Las Vegas (1995) +277 Restoration (1995) +278 Bed of Roses (1996) +279 Once Upon a Time... When We Were Colored (1995) +280 Up Close and Personal (1996) +281 River Wild, The (1994) +282 Time to Kill, A (1996) +283 Emma (1996) +284 Tin Cup (1996) +285 Secrets & Lies (1996) +286 English Patient, The (1996) +287 Marvin's Room (1996) +288 Scream (1996) +289 Evita (1996) +290 Fierce Creatures (1997) +291 Absolute Power (1997) +292 Rosewood (1997) +293 Donnie Brasco (1997) +294 Liar Liar (1997) +295 Breakdown (1997) +296 Promesse, La (1996) +297 Ulee's Gold (1997) +298 Face/Off (1997) +299 Hoodlum (1997) +300 Air Force One (1997) +301 In & Out (1997) +302 L.A. Confidential (1997) +303 Ulee's Gold (1997) +304 Fly Away Home (1996) +305 Ice Storm, The (1997) +306 Mrs. Brown (Her Majesty, Mrs. Brown) (1997) +307 Devil's Advocate, The (1997) +308 FairyTale: A True Story (1997) +309 Deceiver (1997) +310 Rainmaker, The (1997) +311 Wings of the Dove, The (1997) +312 Midnight in the Garden of Good and Evil (1997) +313 Titanic (1997) +314 3 Ninjas: High Noon At Mega Mountain (1998) +315 Apt Pupil (1998) +316 As Good As It Gets (1997) +317 In the Name of the Father (1993) +318 Schindler's List (1993) +319 Everyone Says I Love You (1996) +320 Paradise Lost: The Child Murders at Robin Hood Hills (1996) +321 Mother (1996) +322 Murder at 1600 (1997) +323 Dante's Peak (1997) +324 Lost Highway (1997) +325 Crash (1996) +326 G.I. Jane (1997) +327 Cop Land (1997) +328 Conspiracy Theory (1997) +329 Desperate Measures (1998) +330 187 (1997) +331 Edge, The (1997) +332 Kiss the Girls (1997) +333 Game, The (1997) +334 U Turn (1997) +335 How to Be a Player (1997) +336 Playing God (1997) +337 House of Yes, The (1997) +338 Bean (1997) +339 Mad City (1997) +340 Boogie Nights (1997) +341 Critical Care (1997) +342 Man Who Knew Too Little, The (1997) +343 Alien: Resurrection (1997) +344 Apostle, The (1997) +345 Deconstructing Harry (1997) +346 Jackie Brown (1997) +347 Wag the Dog (1997) +348 Desperate Measures (1998) +349 Hard Rain (1998) +350 Fallen (1998) +351 Prophecy II, The (1998) +352 Spice World (1997) +353 Deep Rising (1998) +354 Wedding Singer, The (1998) +355 Sphere (1998) +356 Client, The (1994) +357 One Flew Over the Cuckoo's Nest (1975) +358 Spawn (1997) +359 Assignment, The (1997) +360 Wonderland (1997) +361 Incognito (1997) +362 Blues Brothers 2000 (1998) +363 Sudden Death (1995) +364 Ace Ventura: When Nature Calls (1995) +365 Powder (1995) +366 Dangerous Minds (1995) +367 Clueless (1995) +368 Bio-Dome (1996) +369 Black Sheep (1996) +370 Mary Reilly (1996) +371 Bridges of Madison County, The (1995) +372 Jeffrey (1995) +373 Judge Dredd (1995) +374 Mighty Morphin Power Rangers: The Movie (1995) +375 Showgirls (1995) +376 Houseguest (1994) +377 Heavyweights (1994) +378 Miracle on 34th Street (1994) +379 Tales From the Crypt Presents: Demon Knight (1995) +380 Star Trek: Generations (1994) +381 Muriel's Wedding (1994) +382 Adventures of Priscilla, Queen of the Desert, The (1994) +383 Flintstones, The (1994) +384 Naked Gun 33 1/3: The Final Insult (1994) +385 True Lies (1994) +386 Addams Family Values (1993) +387 Age of Innocence, The (1993) +388 Beverly Hills Cop III (1994) +389 Black Beauty (1994) +390 Fear of a Black Hat (1993) +391 Last Action Hero (1993) +392 Man Without a Face, The (1993) +393 Mrs. Doubtfire (1993) +394 Radioland Murders (1994) +395 Robin Hood: Men in Tights (1993) +396 Serial Mom (1994) +397 Striking Distance (1993) +398 Super Mario Bros. (1993) +399 Three Musketeers, The (1993) +400 Little Rascals, The (1994) +401 Brady Bunch Movie, The (1995) +402 Ghost (1990) +403 Batman (1989) +404 Pinocchio (1940) +405 Mission: Impossible (1996) +406 Thinner (1996) +407 Spy Hard (1996) +408 Close Shave, A (1995) +409 Jack (1996) +410 Kingpin (1996) +411 Nutty Professor, The (1996) +412 Very Brady Sequel, A (1996) +413 Tales from the Crypt Presents: Bordello of Blood (1996) +414 My Favorite Year (1982) +415 Apple Dumpling Gang, The (1975) +416 Old Yeller (1957) +417 Parent Trap, The (1961) +418 Cinderella (1950) +419 Mary Poppins (1964) +420 Alice in Wonderland (1951) +421 William Shakespeare's Romeo and Juliet (1996) +422 Aladdin and the King of Thieves (1996) +423 E.T. the Extra-Terrestrial (1982) +424 Children of the Corn: The Gathering (1996) +425 Bob Roberts (1992) +426 Transformers: The Movie, The (1986) +427 To Kill a Mockingbird (1962) +428 Harold and Maude (1971) +429 Day the Earth Stood Still, The (1951) +430 Duck Soup (1933) +431 Highlander (1986) +432 Fantasia (1940) +433 Heathers (1989) +434 Forbidden Planet (1956) +435 Butch Cassidy and the Sundance Kid (1969) +436 American Werewolf in London, An (1981) +437 Amityville 1992: It's About Time (1992) +438 Amityville 3-D (1983) +439 Amityville: A New Generation (1993) +440 Amityville II: The Possession (1982) +441 Amityville Horror, The (1979) +442 Amityville Curse, The (1990) +443 Birds, The (1963) +444 Blob, The (1958) +445 Body Snatcher, The (1945) +446 Burnt Offerings (1976) +447 Carrie (1976) +448 Omen, The (1976) +449 Star Trek: The Motion Picture (1979) +450 Star Trek V: The Final Frontier (1989) +451 Grease (1978) +452 Jaws 2 (1978) +453 Jaws 3-D (1983) +454 Bastard Out of Carolina (1996) +455 Jackie Chan's First Strike (1996) +456 Beverly Hills Ninja (1997) +457 Free Willy 3: The Rescue (1997) +458 Nixon (1995) +459 Cry, the Beloved Country (1995) +460 Crossing Guard, The (1995) +461 Smoke (1995) +462 Like Water For Chocolate (Como agua para chocolate) (1992) +463 Secret of Roan Inish, The (1994) +464 Vanya on 42nd Street (1994) +465 Jungle Book, The (1994) +466 Red Rock West (1992) +467 Bronx Tale, A (1993) +468 Rudy (1993) +469 Short Cuts (1993) +470 Tombstone (1993) +471 Courage Under Fire (1996) +472 Dragonheart (1996) +473 James and the Giant Peach (1996) +474 Dr. Strangelove or: How I Learned to Stop Worrying and Love the Bomb (1963) +475 Trainspotting (1996) +476 First Wives Club, The (1996) +477 Matilda (1996) +478 Philadelphia Story, The (1940) +479 Vertigo (1958) +480 North by Northwest (1959) +481 Apartment, The (1960) +482 Some Like It Hot (1959) +483 Casablanca (1942) +484 Maltese Falcon, The (1941) +485 My Fair Lady (1964) +486 Sabrina (1954) +487 Roman Holiday (1953) +488 Sunset Blvd. (1950) +489 Notorious (1946) +490 To Catch a Thief (1955) +491 Adventures of Robin Hood, The (1938) +492 East of Eden (1955) +493 Thin Man, The (1934) +494 His Girl Friday (1940) +495 Around the World in 80 Days (1956) +496 It's a Wonderful Life (1946) +497 Bringing Up Baby (1938) +498 African Queen, The (1951) +499 Cat on a Hot Tin Roof (1958) +500 Fly Away Home (1996) +501 Dumbo (1941) +502 Bananas (1971) +503 Candidate, The (1972) +504 Bonnie and Clyde (1967) +505 Dial M for Murder (1954) +506 Rebel Without a Cause (1955) +507 Streetcar Named Desire, A (1951) +508 People vs. Larry Flynt, The (1996) +509 My Left Foot (1989) +510 Magnificent Seven, The (1954) +511 Lawrence of Arabia (1962) +512 Wings of Desire (1987) +513 Third Man, The (1949) +514 Annie Hall (1977) +515 Boot, Das (1981) +516 Local Hero (1983) +517 Manhattan (1979) +518 Miller's Crossing (1990) +519 Treasure of the Sierra Madre, The (1948) +520 Great Escape, The (1963) +521 Deer Hunter, The (1978) +522 Down by Law (1986) +523 Cool Hand Luke (1967) +524 Great Dictator, The (1940) +525 Big Sleep, The (1946) +526 Ben-Hur (1959) +527 Gandhi (1982) +528 Killing Fields, The (1984) +529 My Life as a Dog (Mitt liv som hund) (1985) +530 Man Who Would Be King, The (1975) +531 Shine (1996) +532 Kama Sutra: A Tale of Love (1996) +533 Daytrippers, The (1996) +534 Traveller (1997) +535 Addicted to Love (1997) +536 Ponette (1996) +537 My Own Private Idaho (1991) +538 Anastasia (1997) +539 Mouse Hunt (1997) +540 Money Train (1995) +541 Mortal Kombat (1995) +542 Pocahontas (1995) +543 Mis�rables, Les (1995) +544 Things to Do in Denver when You're Dead (1995) +545 Vampire in Brooklyn (1995) +546 Broken Arrow (1996) +547 Young Poisoner's Handbook, The (1995) +548 NeverEnding Story III, The (1994) +549 Rob Roy (1995) +550 Die Hard: With a Vengeance (1995) +551 Lord of Illusions (1995) +552 Species (1995) +553 Walk in the Clouds, A (1995) +554 Waterworld (1995) +555 White Man's Burden (1995) +556 Wild Bill (1995) +557 Farinelli: il castrato (1994) +558 Heavenly Creatures (1994) +559 Interview with the Vampire (1994) +560 Kid in King Arthur's Court, A (1995) +561 Mary Shelley's Frankenstein (1994) +562 Quick and the Dead, The (1995) +563 Stephen King's The Langoliers (1995) +564 Tales from the Hood (1995) +565 Village of the Damned (1995) +566 Clear and Present Danger (1994) +567 Wes Craven's New Nightmare (1994) +568 Speed (1994) +569 Wolf (1994) +570 Wyatt Earp (1994) +571 Another Stakeout (1993) +572 Blown Away (1994) +573 Body Snatchers (1993) +574 Boxing Helena (1993) +575 City Slickers II: The Legend of Curly's Gold (1994) +576 Cliffhanger (1993) +577 Coneheads (1993) +578 Demolition Man (1993) +579 Fatal Instinct (1993) +580 Englishman Who Went Up a Hill, But Came Down a Mountain, The (1995) +581 Kalifornia (1993) +582 Piano, The (1993) +583 Romeo Is Bleeding (1993) +584 Secret Garden, The (1993) +585 Son in Law (1993) +586 Terminal Velocity (1994) +587 Hour of the Pig, The (1993) +588 Beauty and the Beast (1991) +589 Wild Bunch, The (1969) +590 Hellraiser: Bloodline (1996) +591 Primal Fear (1996) +592 True Crime (1995) +593 Stalingrad (1993) +594 Heavy (1995) +595 Fan, The (1996) +596 Hunchback of Notre Dame, The (1996) +597 Eraser (1996) +598 Big Squeeze, The (1996) +599 Police Story 4: Project S (Chao ji ji hua) (1993) +600 Daniel Defoe's Robinson Crusoe (1996) +601 For Whom the Bell Tolls (1943) +602 American in Paris, An (1951) +603 Rear Window (1954) +604 It Happened One Night (1934) +605 Meet Me in St. Louis (1944) +606 All About Eve (1950) +607 Rebecca (1940) +608 Spellbound (1945) +609 Father of the Bride (1950) +610 Gigi (1958) +611 Laura (1944) +612 Lost Horizon (1937) +613 My Man Godfrey (1936) +614 Giant (1956) +615 39 Steps, The (1935) +616 Night of the Living Dead (1968) +617 Blue Angel, The (Blaue Engel, Der) (1930) +618 Picnic (1955) +619 Extreme Measures (1996) +620 Chamber, The (1996) +621 Davy Crockett, King of the Wild Frontier (1955) +622 Swiss Family Robinson (1960) +623 Angels in the Outfield (1994) +624 Three Caballeros, The (1945) +625 Sword in the Stone, The (1963) +626 So Dear to My Heart (1949) +627 Robin Hood: Prince of Thieves (1991) +628 Sleepers (1996) +629 Victor/Victoria (1982) +630 Great Race, The (1965) +631 Crying Game, The (1992) +632 Sophie's Choice (1982) +633 Christmas Carol, A (1938) +634 Microcosmos: Le peuple de l'herbe (1996) +635 Fog, The (1980) +636 Escape from New York (1981) +637 Howling, The (1981) +638 Return of Martin Guerre, The (Retour de Martin Guerre, Le) (1982) +639 Tin Drum, The (Blechtrommel, Die) (1979) +640 Cook the Thief His Wife & Her Lover, The (1989) +641 Paths of Glory (1957) +642 Grifters, The (1990) +643 The Innocent (1994) +644 Thin Blue Line, The (1988) +645 Paris Is Burning (1990) +646 Once Upon a Time in the West (1969) +647 Ran (1985) +648 Quiet Man, The (1952) +649 Once Upon a Time in America (1984) +650 Seventh Seal, The (Sjunde inseglet, Det) (1957) +651 Glory (1989) +652 Rosencrantz and Guildenstern Are Dead (1990) +653 Touch of Evil (1958) +654 Chinatown (1974) +655 Stand by Me (1986) +656 M (1931) +657 Manchurian Candidate, The (1962) +658 Pump Up the Volume (1990) +659 Arsenic and Old Lace (1944) +660 Fried Green Tomatoes (1991) +661 High Noon (1952) +662 Somewhere in Time (1980) +663 Being There (1979) +664 Paris, Texas (1984) +665 Alien 3 (1992) +666 Blood For Dracula (Andy Warhol's Dracula) (1974) +667 Audrey Rose (1977) +668 Blood Beach (1981) +669 Body Parts (1991) +670 Body Snatchers (1993) +671 Bride of Frankenstein (1935) +672 Candyman (1992) +673 Cape Fear (1962) +674 Cat People (1982) +675 Nosferatu (Nosferatu, eine Symphonie des Grauens) (1922) +676 Crucible, The (1996) +677 Fire on the Mountain (1996) +678 Volcano (1997) +679 Conan the Barbarian (1981) +680 Kull the Conqueror (1997) +681 Wishmaster (1997) +682 I Know What You Did Last Summer (1997) +683 Rocket Man (1997) +684 In the Line of Fire (1993) +685 Executive Decision (1996) +686 Perfect World, A (1993) +687 McHale's Navy (1997) +688 Leave It to Beaver (1997) +689 Jackal, The (1997) +690 Seven Years in Tibet (1997) +691 Dark City (1998) +692 American President, The (1995) +693 Casino (1995) +694 Persuasion (1995) +695 Kicking and Screaming (1995) +696 City Hall (1996) +697 Basketball Diaries, The (1995) +698 Browning Version, The (1994) +699 Little Women (1994) +700 Miami Rhapsody (1995) +701 Wonderful, Horrible Life of Leni Riefenstahl, The (1993) +702 Barcelona (1994) +703 Widows' Peak (1994) +704 House of the Spirits, The (1993) +705 Singin' in the Rain (1952) +706 Bad Moon (1996) +707 Enchanted April (1991) +708 Sex, Lies, and Videotape (1989) +709 Strictly Ballroom (1992) +710 Better Off Dead... (1985) +711 Substance of Fire, The (1996) +712 Tin Men (1987) +713 Othello (1995) +714 Carrington (1995) +715 To Die For (1995) +716 Home for the Holidays (1995) +717 Juror, The (1996) +718 In the Bleak Midwinter (1995) +719 Canadian Bacon (1994) +720 First Knight (1995) +721 Mallrats (1995) +722 Nine Months (1995) +723 Boys on the Side (1995) +724 Circle of Friends (1995) +725 Exit to Eden (1994) +726 Fluke (1995) +727 Immortal Beloved (1994) +728 Junior (1994) +729 Nell (1994) +730 Queen Margot (Reine Margot, La) (1994) +731 Corrina, Corrina (1994) +732 Dave (1993) +733 Go Fish (1994) +734 Made in America (1993) +735 Philadelphia (1993) +736 Shadowlands (1993) +737 Sirens (1994) +738 Threesome (1994) +739 Pretty Woman (1990) +740 Jane Eyre (1996) +741 Last Supper, The (1995) +742 Ransom (1996) +743 Crow: City of Angels, The (1996) +744 Michael Collins (1996) +745 Ruling Class, The (1972) +746 Real Genius (1985) +747 Benny & Joon (1993) +748 Saint, The (1997) +749 MatchMaker, The (1997) +750 Amistad (1997) +751 Tomorrow Never Dies (1997) +752 Replacement Killers, The (1998) +753 Burnt By the Sun (1994) +754 Red Corner (1997) +755 Jumanji (1995) +756 Father of the Bride Part II (1995) +757 Across the Sea of Time (1995) +758 Lawnmower Man 2: Beyond Cyberspace (1996) +759 Fair Game (1995) +760 Screamers (1995) +761 Nick of Time (1995) +762 Beautiful Girls (1996) +763 Happy Gilmore (1996) +764 If Lucy Fell (1996) +765 Boomerang (1992) +766 Man of the Year (1995) +767 Addiction, The (1995) +768 Casper (1995) +769 Congo (1995) +770 Devil in a Blue Dress (1995) +771 Johnny Mnemonic (1995) +772 Kids (1995) +773 Mute Witness (1994) +774 Prophecy, The (1995) +775 Something to Talk About (1995) +776 Three Wishes (1995) +777 Castle Freak (1995) +778 Don Juan DeMarco (1995) +779 Drop Zone (1994) +780 Dumb & Dumber (1994) +781 French Kiss (1995) +782 Little Odessa (1994) +783 Milk Money (1994) +784 Beyond Bedlam (1993) +785 Only You (1994) +786 Perez Family, The (1995) +787 Roommates (1995) +788 Relative Fear (1994) +789 Swimming with Sharks (1995) +790 Tommy Boy (1995) +791 Baby-Sitters Club, The (1995) +792 Bullets Over Broadway (1994) +793 Crooklyn (1994) +794 It Could Happen to You (1994) +795 Richie Rich (1994) +796 Speechless (1994) +797 Timecop (1994) +798 Bad Company (1995) +799 Boys Life (1995) +800 In the Mouth of Madness (1995) +801 Air Up There, The (1994) +802 Hard Target (1993) +803 Heaven & Earth (1993) +804 Jimmy Hollywood (1994) +805 Manhattan Murder Mystery (1993) +806 Menace II Society (1993) +807 Poetic Justice (1993) +808 Program, The (1993) +809 Rising Sun (1993) +810 Shadow, The (1994) +811 Thirty-Two Short Films About Glenn Gould (1993) +812 Andre (1994) +813 Celluloid Closet, The (1995) +814 Great Day in Harlem, A (1994) +815 One Fine Day (1996) +816 Candyman: Farewell to the Flesh (1995) +817 Frisk (1995) +818 Girl 6 (1996) +819 Eddie (1996) +820 Space Jam (1996) +821 Mrs. Winterbourne (1996) +822 Faces (1968) +823 Mulholland Falls (1996) +824 Great White Hype, The (1996) +825 Arrival, The (1996) +826 Phantom, The (1996) +827 Daylight (1996) +828 Alaska (1996) +829 Fled (1996) +830 Power 98 (1995) +831 Escape from L.A. (1996) +832 Bogus (1996) +833 Bulletproof (1996) +834 Halloween: The Curse of Michael Myers (1995) +835 Gay Divorcee, The (1934) +836 Ninotchka (1939) +837 Meet John Doe (1941) +838 In the Line of Duty 2 (1987) +839 Loch Ness (1995) +840 Last Man Standing (1996) +841 Glimmer Man, The (1996) +842 Pollyanna (1960) +843 Shaggy Dog, The (1959) +844 Freeway (1996) +845 That Thing You Do! (1996) +846 To Gillian on Her 37th Birthday (1996) +847 Looking for Richard (1996) +848 Murder, My Sweet (1944) +849 Days of Thunder (1990) +850 Perfect Candidate, A (1996) +851 Two or Three Things I Know About Her (1966) +852 Bloody Child, The (1996) +853 Braindead (1992) +854 Bad Taste (1987) +855 Diva (1981) +856 Night on Earth (1991) +857 Paris Was a Woman (1995) +858 Amityville: Dollhouse (1996) +859 April Fool's Day (1986) +860 Believers, The (1987) +861 Nosferatu a Venezia (1986) +862 Jingle All the Way (1996) +863 Garden of Finzi-Contini, The (Giardino dei Finzi-Contini, Il) (1970) +864 My Fellow Americans (1996) +865 Ice Storm, The (1997) +866 Michael (1996) +867 Whole Wide World, The (1996) +868 Hearts and Minds (1996) +869 Fools Rush In (1997) +870 Touch (1997) +871 Vegas Vacation (1997) +872 Love Jones (1997) +873 Picture Perfect (1997) +874 Career Girls (1997) +875 She's So Lovely (1997) +876 Money Talks (1997) +877 Excess Baggage (1997) +878 That Darn Cat! (1997) +879 Peacemaker, The (1997) +880 Soul Food (1997) +881 Money Talks (1997) +882 Washington Square (1997) +883 Telling Lies in America (1997) +884 Year of the Horse (1997) +885 Phantoms (1998) +886 Life Less Ordinary, A (1997) +887 Eve's Bayou (1997) +888 One Night Stand (1997) +889 Tango Lesson, The (1997) +890 Mortal Kombat: Annihilation (1997) +891 Bent (1997) +892 Flubber (1997) +893 For Richer or Poorer (1997) +894 Home Alone 3 (1997) +895 Scream 2 (1997) +896 Sweet Hereafter, The (1997) +897 Time Tracers (1995) +898 Postman, The (1997) +899 Winter Guest, The (1997) +900 Kundun (1997) +901 Mr. Magoo (1997) +902 Big Lebowski, The (1998) +903 Afterglow (1997) +904 Ma vie en rose (My Life in Pink) (1997) +905 Great Expectations (1998) +906 Oscar & Lucinda (1997) +907 Vermin (1998) +908 Half Baked (1998) +909 Dangerous Beauty (1998) +910 Nil By Mouth (1997) +911 Twilight (1998) +912 U.S. Marshalls (1998) +913 Love and Death on Long Island (1997) +914 Wild Things (1998) +915 Primary Colors (1998) +916 Lost in Space (1998) +917 Mercury Rising (1998) +918 City of Angels (1998) +919 City of Lost Children, The (1995) +920 Two Bits (1995) +921 Farewell My Concubine (1993) +922 Dead Man (1995) +923 Raise the Red Lantern (1991) +924 White Squall (1996) +925 Unforgettable (1996) +926 Down Periscope (1996) +927 Flower of My Secret, The (Flor de mi secreto, La) (1995) +928 Craft, The (1996) +929 Harriet the Spy (1996) +930 Chain Reaction (1996) +931 Island of Dr. Moreau, The (1996) +932 First Kid (1996) +933 Funeral, The (1996) +934 Preacher's Wife, The (1996) +935 Paradise Road (1997) +936 Brassed Off (1996) +937 Thousand Acres, A (1997) +938 Smile Like Yours, A (1997) +939 Murder in the First (1995) +940 Airheads (1994) +941 With Honors (1994) +942 What's Love Got to Do with It (1993) +943 Killing Zoe (1994) +944 Renaissance Man (1994) +945 Charade (1963) +946 Fox and the Hound, The (1981) +947 Big Blue, The (Grand bleu, Le) (1988) +948 Booty Call (1997) +949 How to Make an American Quilt (1995) +950 Georgia (1995) +951 Indian in the Cupboard, The (1995) +952 Blue in the Face (1995) +953 Unstrung Heroes (1995) +954 Unzipped (1995) +955 Before Sunrise (1995) +956 Nobody's Fool (1994) +957 Pushing Hands (1992) +958 To Live (Huozhe) (1994) +959 Dazed and Confused (1993) +960 Naked (1993) +961 Orlando (1993) +962 Ruby in Paradise (1993) +963 Some Folks Call It a Sling Blade (1993) +964 Month by the Lake, A (1995) +965 Funny Face (1957) +966 Affair to Remember, An (1957) +967 Little Lord Fauntleroy (1936) +968 Inspector General, The (1949) +969 Winnie the Pooh and the Blustery Day (1968) +970 Hear My Song (1991) +971 Mediterraneo (1991) +972 Passion Fish (1992) +973 Grateful Dead (1995) +974 Eye for an Eye (1996) +975 Fear (1996) +976 Solo (1996) +977 Substitute, The (1996) +978 Heaven's Prisoners (1996) +979 Trigger Effect, The (1996) +980 Mother Night (1996) +981 Dangerous Ground (1997) +982 Maximum Risk (1996) +983 Rich Man's Wife, The (1996) +984 Shadow Conspiracy (1997) +985 Blood & Wine (1997) +986 Turbulence (1997) +987 Underworld (1997) +988 Beautician and the Beast, The (1997) +989 Cats Don't Dance (1997) +990 Anna Karenina (1997) +991 Keys to Tulsa (1997) +992 Head Above Water (1996) +993 Hercules (1997) +994 Last Time I Committed Suicide, The (1997) +995 Kiss Me, Guido (1997) +996 Big Green, The (1995) +997 Stuart Saves His Family (1995) +998 Cabin Boy (1994) +999 Clean Slate (1994) +1000 Lightning Jack (1994) +1001 Stupids, The (1996) +1002 Pest, The (1997) +1003 That Darn Cat! (1997) +1004 Geronimo: An American Legend (1993) +1005 Double vie de V�ronique, La (Double Life of Veronique, The) (1991) +1006 Until the End of the World (Bis ans Ende der Welt) (1991) +1007 Waiting for Guffman (1996) +1008 I Shot Andy Warhol (1996) +1009 Stealing Beauty (1996) +1010 Basquiat (1996) +1011 2 Days in the Valley (1996) +1012 Private Parts (1997) +1013 Anaconda (1997) +1014 Romy and Michele's High School Reunion (1997) +1015 Shiloh (1997) +1016 Con Air (1997) +1017 Trees Lounge (1996) +1018 Tie Me Up! Tie Me Down! (1990) +1019 Die xue shuang xiong (Killer, The) (1989) +1020 Gaslight (1944) +1021 8 1/2 (1963) +1022 Fast, Cheap & Out of Control (1997) +1023 Fathers' Day (1997) +1024 Mrs. Dalloway (1997) +1025 Fire Down Below (1997) +1026 Lay of the Land, The (1997) +1027 Shooter, The (1995) +1028 Grumpier Old Men (1995) +1029 Jury Duty (1995) +1030 Beverly Hillbillies, The (1993) +1031 Lassie (1994) +1032 Little Big League (1994) +1033 Homeward Bound II: Lost in San Francisco (1996) +1034 Quest, The (1996) +1035 Cool Runnings (1993) +1036 Drop Dead Fred (1991) +1037 Grease 2 (1982) +1038 Switchback (1997) +1039 Hamlet (1996) +1040 Two if by Sea (1996) +1041 Forget Paris (1995) +1042 Just Cause (1995) +1043 Rent-a-Kid (1995) +1044 Paper, The (1994) +1045 Fearless (1993) +1046 Malice (1993) +1047 Multiplicity (1996) +1048 She's the One (1996) +1049 House Arrest (1996) +1050 Ghost and Mrs. Muir, The (1947) +1051 Associate, The (1996) +1052 Dracula: Dead and Loving It (1995) +1053 Now and Then (1995) +1054 Mr. Wrong (1996) +1055 Simple Twist of Fate, A (1994) +1056 Cronos (1992) +1057 Pallbearer, The (1996) +1058 War, The (1994) +1059 Don't Be a Menace to South Central While Drinking Your Juice in the Hood (1996) +1060 Adventures of Pinocchio, The (1996) +1061 Evening Star, The (1996) +1062 Four Days in September (1997) +1063 Little Princess, A (1995) +1064 Crossfire (1947) +1065 Koyaanisqatsi (1983) +1066 Balto (1995) +1067 Bottle Rocket (1996) +1068 Star Maker, The (Uomo delle stelle, L') (1995) +1069 Amateur (1994) +1070 Living in Oblivion (1995) +1071 Party Girl (1995) +1072 Pyromaniac's Love Story, A (1995) +1073 Shallow Grave (1994) +1074 Reality Bites (1994) +1075 Man of No Importance, A (1994) +1076 Pagemaster, The (1994) +1077 Love and a .45 (1994) +1078 Oliver & Company (1988) +1079 Joe's Apartment (1996) +1080 Celestial Clockwork (1994) +1081 Curdled (1996) +1082 Female Perversions (1996) +1083 Albino Alligator (1996) +1084 Anne Frank Remembered (1995) +1085 Carried Away (1996) +1086 It's My Party (1995) +1087 Bloodsport 2 (1995) +1088 Double Team (1997) +1089 Speed 2: Cruise Control (1997) +1090 Sliver (1993) +1091 Pete's Dragon (1977) +1092 Dear God (1996) +1093 Live Nude Girls (1995) +1094 Thin Line Between Love and Hate, A (1996) +1095 High School High (1996) +1096 Commandments (1997) +1097 Hate (Haine, La) (1995) +1098 Flirting With Disaster (1996) +1099 Red Firecracker, Green Firecracker (1994) +1100 What Happened Was... (1994) +1101 Six Degrees of Separation (1993) +1102 Two Much (1996) +1103 Trust (1990) +1104 C'est arriv� pr�s de chez vous (1992) +1105 Firestorm (1998) +1106 Newton Boys, The (1998) +1107 Beyond Rangoon (1995) +1108 Feast of July (1995) +1109 Death and the Maiden (1994) +1110 Tank Girl (1995) +1111 Double Happiness (1994) +1112 Cobb (1994) +1113 Mrs. Parker and the Vicious Circle (1994) +1114 Faithful (1996) +1115 Twelfth Night (1996) +1116 Mark of Zorro, The (1940) +1117 Surviving Picasso (1996) +1118 Up in Smoke (1978) +1119 Some Kind of Wonderful (1987) +1120 I'm Not Rappaport (1996) +1121 Umbrellas of Cherbourg, The (Parapluies de Cherbourg, Les) (1964) +1122 They Made Me a Criminal (1939) +1123 Last Time I Saw Paris, The (1954) +1124 Farewell to Arms, A (1932) +1125 Innocents, The (1961) +1126 Old Man and the Sea, The (1958) +1127 Truman Show, The (1998) +1128 Heidi Fleiss: Hollywood Madam (1995) +1129 Chungking Express (1994) +1130 Jupiter's Wife (1994) +1131 Safe (1995) +1132 Feeling Minnesota (1996) +1133 Escape to Witch Mountain (1975) +1134 Get on the Bus (1996) +1135 Doors, The (1991) +1136 Ghosts of Mississippi (1996) +1137 Beautiful Thing (1996) +1138 Best Men (1997) +1139 Hackers (1995) +1140 Road to Wellville, The (1994) +1141 War Room, The (1993) +1142 When We Were Kings (1996) +1143 Hard Eight (1996) +1144 Quiet Room, The (1996) +1145 Blue Chips (1994) +1146 Calendar Girl (1993) +1147 My Family (1995) +1148 Tom & Viv (1994) +1149 Walkabout (1971) +1150 Last Dance (1996) +1151 Original Gangstas (1996) +1152 In Love and War (1996) +1153 Backbeat (1993) +1154 Alphaville (1965) +1155 Rendezvous in Paris (Rendez-vous de Paris, Les) (1995) +1156 Cyclo (1995) +1157 Relic, The (1997) +1158 Fille seule, La (A Single Girl) (1995) +1159 Stalker (1979) +1160 Love! Valour! Compassion! (1997) +1161 Palookaville (1996) +1162 Phat Beach (1996) +1163 Portrait of a Lady, The (1996) +1164 Zeus and Roxanne (1997) +1165 Big Bully (1996) +1166 Love & Human Remains (1993) +1167 Sum of Us, The (1994) +1168 Little Buddha (1993) +1169 Fresh (1994) +1170 Spanking the Monkey (1994) +1171 Wild Reeds (1994) +1172 Women, The (1939) +1173 Bliss (1997) +1174 Caught (1996) +1175 Hugo Pool (1997) +1176 Welcome To Sarajevo (1997) +1177 Dunston Checks In (1996) +1178 Major Payne (1994) +1179 Man of the House (1995) +1180 I Love Trouble (1994) +1181 Low Down Dirty Shame, A (1994) +1182 Cops and Robbersons (1994) +1183 Cowboy Way, The (1994) +1184 Endless Summer 2, The (1994) +1185 In the Army Now (1994) +1186 Inkwell, The (1994) +1187 Switchblade Sisters (1975) +1188 Young Guns II (1990) +1189 Prefontaine (1997) +1190 That Old Feeling (1997) +1191 Letter From Death Row, A (1998) +1192 Boys of St. Vincent, The (1993) +1193 Before the Rain (Pred dozhdot) (1994) +1194 Once Were Warriors (1994) +1195 Strawberry and Chocolate (Fresa y chocolate) (1993) +1196 Savage Nights (Nuits fauves, Les) (1992) +1197 Family Thing, A (1996) +1198 Purple Noon (1960) +1199 Cemetery Man (Dellamorte Dellamore) (1994) +1200 Kim (1950) +1201 Marlene Dietrich: Shadow and Light (1996) +1202 Maybe, Maybe Not (Bewegte Mann, Der) (1994) +1203 Top Hat (1935) +1204 To Be or Not to Be (1942) +1205 Secret Agent, The (1996) +1206 Amos & Andrew (1993) +1207 Jade (1995) +1208 Kiss of Death (1995) +1209 Mixed Nuts (1994) +1210 Virtuosity (1995) +1211 Blue Sky (1994) +1212 Flesh and Bone (1993) +1213 Guilty as Sin (1993) +1214 In the Realm of the Senses (Ai no corrida) (1976) +1215 Barb Wire (1996) +1216 Kissed (1996) +1217 Assassins (1995) +1218 Friday (1995) +1219 Goofy Movie, A (1995) +1220 Higher Learning (1995) +1221 When a Man Loves a Woman (1994) +1222 Judgment Night (1993) +1223 King of the Hill (1993) +1224 Scout, The (1994) +1225 Angus (1995) +1226 Night Falls on Manhattan (1997) +1227 Awfully Big Adventure, An (1995) +1228 Under Siege 2: Dark Territory (1995) +1229 Poison Ivy II (1995) +1230 Ready to Wear (Pret-A-Porter) (1994) +1231 Marked for Death (1990) +1232 Madonna: Truth or Dare (1991) +1233 N�nette et Boni (1996) +1234 Chairman of the Board (1998) +1235 Big Bang Theory, The (1994) +1236 Other Voices, Other Rooms (1997) +1237 Twisted (1996) +1238 Full Speed (1996) +1239 Cutthroat Island (1995) +1240 Ghost in the Shell (Kokaku kidotai) (1995) +1241 Van, The (1996) +1242 Old Lady Who Walked in the Sea, The (Vieille qui marchait dans la mer, La) (1991) +1243 Night Flier (1997) +1244 Metro (1997) +1245 Gridlock'd (1997) +1246 Bushwhacked (1995) +1247 Bad Girls (1994) +1248 Blink (1994) +1249 For Love or Money (1993) +1250 Best of the Best 3: No Turning Back (1995) +1251 A Chef in Love (1996) +1252 Contempt (M�pris, Le) (1963) +1253 Tie That Binds, The (1995) +1254 Gone Fishin' (1997) +1255 Broken English (1996) +1256 Designated Mourner, The (1997) +1257 Designated Mourner, The (1997) +1258 Trial and Error (1997) +1259 Pie in the Sky (1995) +1260 Total Eclipse (1995) +1261 Run of the Country, The (1995) +1262 Walking and Talking (1996) +1263 Foxfire (1996) +1264 Nothing to Lose (1994) +1265 Star Maps (1997) +1266 Bread and Chocolate (Pane e cioccolata) (1973) +1267 Clockers (1995) +1268 Bitter Moon (1992) +1269 Love in the Afternoon (1957) +1270 Life with Mikey (1993) +1271 North (1994) +1272 Talking About Sex (1994) +1273 Color of Night (1994) +1274 Robocop 3 (1993) +1275 Killer (Bulletproof Heart) (1994) +1276 Sunset Park (1996) +1277 Set It Off (1996) +1278 Selena (1997) +1279 Wild America (1997) +1280 Gang Related (1997) +1281 Manny & Lo (1996) +1282 Grass Harp, The (1995) +1283 Out to Sea (1997) +1284 Before and After (1996) +1285 Princess Caraboo (1994) +1286 Shall We Dance? (1937) +1287 Ed (1996) +1288 Denise Calls Up (1995) +1289 Jack and Sarah (1995) +1290 Country Life (1994) +1291 Celtic Pride (1996) +1292 Simple Wish, A (1997) +1293 Star Kid (1997) +1294 Ayn Rand: A Sense of Life (1997) +1295 Kicked in the Head (1997) +1296 Indian Summer (1996) +1297 Love Affair (1994) +1298 Band Wagon, The (1953) +1299 Penny Serenade (1941) +1300 'Til There Was You (1997) +1301 Stripes (1981) +1302 Late Bloomers (1996) +1303 Getaway, The (1994) +1304 New York Cop (1996) +1305 National Lampoon's Senior Trip (1995) +1306 Delta of Venus (1994) +1307 Carmen Miranda: Bananas Is My Business (1994) +1308 Babyfever (1994) +1309 Very Natural Thing, A (1974) +1310 Walk in the Sun, A (1945) +1311 Waiting to Exhale (1995) +1312 Pompatus of Love, The (1996) +1313 Palmetto (1998) +1314 Surviving the Game (1994) +1315 Inventing the Abbotts (1997) +1316 Horse Whisperer, The (1998) +1317 Journey of August King, The (1995) +1318 Catwalk (1995) +1319 Neon Bible, The (1995) +1320 Homage (1995) +1321 Open Season (1996) +1322 Metisse (Caf� au Lait) (1993) +1323 Wooden Man's Bride, The (Wu Kui) (1994) +1324 Loaded (1994) +1325 August (1996) +1326 Boys (1996) +1327 Captives (1994) +1328 Of Love and Shadows (1994) +1329 Low Life, The (1994) +1330 An Unforgettable Summer (1994) +1331 Last Klezmer: Leopold Kozlowski, His Life and Music, The (1995) +1332 My Life and Times With Antonin Artaud (En compagnie d'Antonin Artaud) (1993) +1333 Midnight Dancers (Sibak) (1994) +1334 Somebody to Love (1994) +1335 American Buffalo (1996) +1336 Kazaam (1996) +1337 Larger Than Life (1996) +1338 Two Deaths (1995) +1339 Stefano Quantestorie (1993) +1340 Crude Oasis, The (1995) +1341 Hedd Wyn (1992) +1342 Convent, The (Convento, O) (1995) +1343 Lotto Land (1995) +1344 Story of Xinghua, The (1993) +1345 Day the Sun Turned Cold, The (Tianguo niezi) (1994) +1346 Dingo (1992) +1347 Ballad of Narayama, The (Narayama Bushiko) (1958) +1348 Every Other Weekend (1990) +1349 Mille bolle blu (1993) +1350 Crows and Sparrows (1949) +1351 Lover's Knot (1996) +1352 Shadow of Angels (Schatten der Engel) (1976) +1353 1-900 (1994) +1354 Venice/Venice (1992) +1355 Infinity (1996) +1356 Ed's Next Move (1996) +1357 For the Moment (1994) +1358 The Deadly Cure (1996) +1359 Boys in Venice (1996) +1360 Sexual Life of the Belgians, The (1994) +1361 Search for One-eye Jimmy, The (1996) +1362 American Strays (1996) +1363 Leopard Son, The (1996) +1364 Bird of Prey (1996) +1365 Johnny 100 Pesos (1993) +1366 JLG/JLG - autoportrait de d�cembre (1994) +1367 Faust (1994) +1368 Mina Tannenbaum (1994) +1369 Forbidden Christ, The (Cristo proibito, Il) (1950) +1370 I Can't Sleep (J'ai pas sommeil) (1994) +1371 Machine, The (1994) +1372 Stranger, The (1994) +1373 Good Morning (1971) +1374 Falling in Love Again (1980) +1375 Cement Garden, The (1993) +1376 Meet Wally Sparks (1997) +1377 Hotel de Love (1996) +1378 Rhyme & Reason (1997) +1379 Love and Other Catastrophes (1996) +1380 Hollow Reed (1996) +1381 Losing Chase (1996) +1382 Bonheur, Le (1965) +1383 Second Jungle Book: Mowgli & Baloo, The (1997) +1384 Squeeze (1996) +1385 Roseanna's Grave (For Roseanna) (1997) +1386 Tetsuo II: Body Hammer (1992) +1387 Fall (1997) +1388 Gabbeh (1996) +1389 Mondo (1996) +1390 Innocent Sleep, The (1995) +1391 For Ever Mozart (1996) +1392 Locusts, The (1997) +1393 Stag (1997) +1394 Swept from the Sea (1997) +1395 Hurricane Streets (1998) +1396 Stonewall (1995) +1397 Of Human Bondage (1934) +1398 Anna (1996) +1399 Stranger in the House (1997) +1400 Picture Bride (1995) +1401 M. Butterfly (1993) +1402 Ciao, Professore! (1993) +1403 Caro Diario (Dear Diary) (1994) +1404 Withnail and I (1987) +1405 Boy's Life 2 (1997) +1406 When Night Is Falling (1995) +1407 Specialist, The (1994) +1408 Gordy (1995) +1409 Swan Princess, The (1994) +1410 Harlem (1993) +1411 Barbarella (1968) +1412 Land Before Time III: The Time of the Great Giving (1995) (V) +1413 Street Fighter (1994) +1414 Coldblooded (1995) +1415 Next Karate Kid, The (1994) +1416 No Escape (1994) +1417 Turning, The (1992) +1418 Joy Luck Club, The (1993) +1419 Highlander III: The Sorcerer (1994) +1420 Gilligan's Island: The Movie (1998) +1421 My Crazy Life (Mi vida loca) (1993) +1422 Suture (1993) +1423 Walking Dead, The (1995) +1424 I Like It Like That (1994) +1425 I'll Do Anything (1994) +1426 Grace of My Heart (1996) +1427 Drunks (1995) +1428 SubUrbia (1997) +1429 Sliding Doors (1998) +1430 Ill Gotten Gains (1997) +1431 Legal Deceit (1997) +1432 Mighty, The (1998) +1433 Men of Means (1998) +1434 Shooting Fish (1997) +1435 Steal Big, Steal Little (1995) +1436 Mr. Jones (1993) +1437 House Party 3 (1994) +1438 Panther (1995) +1439 Jason's Lyric (1994) +1440 Above the Rim (1994) +1441 Moonlight and Valentino (1995) +1442 Scarlet Letter, The (1995) +1443 8 Seconds (1994) +1444 That Darn Cat! (1965) +1445 Ladybird Ladybird (1994) +1446 Bye Bye, Love (1995) +1447 Century (1993) +1448 My Favorite Season (1993) +1449 Pather Panchali (1955) +1450 Golden Earrings (1947) +1451 Foreign Correspondent (1940) +1452 Lady of Burlesque (1943) +1453 Angel on My Shoulder (1946) +1454 Angel and the Badman (1947) +1455 Outlaw, The (1943) +1456 Beat the Devil (1954) +1457 Love Is All There Is (1996) +1458 Damsel in Distress, A (1937) +1459 Madame Butterfly (1995) +1460 Sleepover (1995) +1461 Here Comes Cookie (1935) +1462 Thieves (Voleurs, Les) (1996) +1463 Boys, Les (1997) +1464 Stars Fell on Henrietta, The (1995) +1465 Last Summer in the Hamptons (1995) +1466 Margaret's Museum (1995) +1467 Saint of Fort Washington, The (1993) +1468 Cure, The (1995) +1469 Tom and Huck (1995) +1470 Gumby: The Movie (1995) +1471 Hideaway (1995) +1472 Visitors, The (Visiteurs, Les) (1993) +1473 Little Princess, The (1939) +1474 Nina Takes a Lover (1994) +1475 Bhaji on the Beach (1993) +1476 Raw Deal (1948) +1477 Nightwatch (1997) +1478 Dead Presidents (1995) +1479 Reckless (1995) +1480 Herbie Rides Again (1974) +1481 S.F.W. (1994) +1482 Gate of Heavenly Peace, The (1995) +1483 Man in the Iron Mask, The (1998) +1484 Jerky Boys, The (1994) +1485 Colonel Chabert, Le (1994) +1486 Girl in the Cadillac (1995) +1487 Even Cowgirls Get the Blues (1993) +1488 Germinal (1993) +1489 Chasers (1994) +1490 Fausto (1993) +1491 Tough and Deadly (1995) +1492 Window to Paris (1994) +1493 Modern Affair, A (1995) +1494 Mostro, Il (1994) +1495 Flirt (1995) +1496 Carpool (1996) +1497 Line King: Al Hirschfeld, The (1996) +1498 Farmer & Chase (1995) +1499 Grosse Fatigue (1994) +1500 Santa with Muscles (1996) +1501 Prisoner of the Mountains (Kavkazsky Plennik) (1996) +1502 Naked in New York (1994) +1503 Gold Diggers: The Secret of Bear Mountain (1995) +1504 Bewegte Mann, Der (1994) +1505 Killer: A Journal of Murder (1995) +1506 Nelly & Monsieur Arnaud (1995) +1507 Three Lives and Only One Death (1996) +1508 Babysitter, The (1995) +1509 Getting Even with Dad (1994) +1510 Mad Dog Time (1996) +1511 Children of the Revolution (1996) +1512 World of Apu, The (Apur Sansar) (1959) +1513 Sprung (1997) +1514 Dream With the Fishes (1997) +1515 Wings of Courage (1995) +1516 Wedding Gift, The (1994) +1517 Race the Sun (1996) +1518 Losing Isaiah (1995) +1519 New Jersey Drive (1995) +1520 Fear, The (1995) +1521 Mr. Wonderful (1993) +1522 Trial by Jury (1994) +1523 Good Man in Africa, A (1994) +1524 Kaspar Hauser (1993) +1525 Object of My Affection, The (1998) +1526 Witness (1985) +1527 Senseless (1998) +1528 Nowhere (1997) +1529 Underground (1995) +1530 Jefferson in Paris (1995) +1531 Far From Home: The Adventures of Yellow Dog (1995) +1532 Foreign Student (1994) +1533 I Don't Want to Talk About It (De eso no se habla) (1993) +1534 Twin Town (1997) +1535 Enfer, L' (1994) +1536 Aiqing wansui (1994) +1537 Cosi (1996) +1538 All Over Me (1997) +1539 Being Human (1993) +1540 Amazing Panda Adventure, The (1995) +1541 Beans of Egypt, Maine, The (1994) +1542 Scarlet Letter, The (1926) +1543 Johns (1996) +1544 It Takes Two (1995) +1545 Frankie Starlight (1995) +1546 Shadows (Cienie) (1988) +1547 Show, The (1995) +1548 The Courtyard (1995) +1549 Dream Man (1995) +1550 Destiny Turns on the Radio (1995) +1551 Glass Shield, The (1994) +1552 Hunted, The (1995) +1553 Underneath, The (1995) +1554 Safe Passage (1994) +1555 Secret Adventures of Tom Thumb, The (1993) +1556 Condition Red (1995) +1557 Yankee Zulu (1994) +1558 Aparajito (1956) +1559 Hostile Intentions (1994) +1560 Clean Slate (Coup de Torchon) (1981) +1561 Tigrero: A Film That Was Never Made (1994) +1562 Eye of Vichy, The (Oeil de Vichy, L') (1993) +1563 Promise, The (Versprechen, Das) (1994) +1564 To Cross the Rubicon (1991) +1565 Daens (1992) +1566 Man from Down Under, The (1943) +1567 Careful (1992) +1568 Vermont Is For Lovers (1992) +1569 Vie est belle, La (Life is Rosey) (1987) +1570 Quartier Mozart (1992) +1571 Touki Bouki (Journey of the Hyena) (1973) +1572 Wend Kuuni (God's Gift) (1982) +1573 Spirits of the Dead (Tre passi nel delirio) (1968) +1574 Pharaoh's Army (1995) +1575 I, Worst of All (Yo, la peor de todas) (1990) +1576 Hungarian Fairy Tale, A (1987) +1577 Death in the Garden (Mort en ce jardin, La) (1956) +1578 Collectionneuse, La (1967) +1579 Baton Rouge (1988) +1580 Liebelei (1933) +1581 Woman in Question, The (1950) +1582 T-Men (1947) +1583 Invitation, The (Zaproszenie) (1986) +1584 Symphonie pastorale, La (1946) +1585 American Dream (1990) +1586 Lashou shentan (1992) +1587 Terror in a Texas Town (1958) +1588 Salut cousin! (1996) +1589 Schizopolis (1996) +1590 To Have, or Not (1995) +1591 Duoluo tianshi (1995) +1592 Magic Hour, The (1998) +1593 Death in Brunswick (1991) +1594 Everest (1998) +1595 Shopping (1994) +1596 Nemesis 2: Nebula (1995) +1597 Romper Stomper (1992) +1598 City of Industry (1997) +1599 Someone Else's America (1995) +1600 Guantanamera (1994) +1601 Office Killer (1997) +1602 Price Above Rubies, A (1998) +1603 Angela (1995) +1604 He Walked by Night (1948) +1605 Love Serenade (1996) +1606 Deceiver (1997) +1607 Hurricane Streets (1998) +1608 Buddy (1997) +1609 B*A*P*S (1997) +1610 Truth or Consequences, N.M. (1997) +1611 Intimate Relations (1996) +1612 Leading Man, The (1996) +1613 Tokyo Fist (1995) +1614 Reluctant Debutante, The (1958) +1615 Warriors of Virtue (1997) +1616 Desert Winds (1995) +1617 Hugo Pool (1997) +1618 King of New York (1990) +1619 All Things Fair (1996) +1620 Sixth Man, The (1997) +1621 Butterfly Kiss (1995) +1622 Paris, France (1993) +1623 C�r�monie, La (1995) +1624 Hush (1998) +1625 Nightwatch (1997) +1626 Nobody Loves Me (Keiner liebt mich) (1994) +1627 Wife, The (1995) +1628 Lamerica (1994) +1629 Nico Icon (1995) +1630 Silence of the Palace, The (Saimt el Qusur) (1994) +1631 Slingshot, The (1993) +1632 Land and Freedom (Tierra y libertad) (1995) +1633 � k�ldum klaka (Cold Fever) (1994) +1634 Etz Hadomim Tafus (Under the Domin Tree) (1994) +1635 Two Friends (1986) +1636 Brothers in Trouble (1995) +1637 Girls Town (1996) +1638 Normal Life (1996) +1639 Bitter Sugar (Azucar Amargo) (1996) +1640 Eighth Day, The (1996) +1641 Dadetown (1995) +1642 Some Mother's Son (1996) +1643 Angel Baby (1995) +1644 Sudden Manhattan (1996) +1645 Butcher Boy, The (1998) +1646 Men With Guns (1997) +1647 Hana-bi (1997) +1648 Niagara, Niagara (1997) +1649 Big One, The (1997) +1650 Butcher Boy, The (1998) +1651 Spanish Prisoner, The (1997) +1652 Temptress Moon (Feng Yue) (1996) +1653 Entertaining Angels: The Dorothy Day Story (1996) +1654 Chairman of the Board (1998) +1655 Favor, The (1994) +1656 Little City (1998) +1657 Target (1995) +1658 Substance of Fire, The (1996) +1659 Getting Away With Murder (1996) +1660 Small Faces (1995) +1661 New Age, The (1994) +1662 Rough Magic (1995) +1663 Nothing Personal (1995) +1664 8 Heads in a Duffel Bag (1997) +1665 Brother's Kiss, A (1997) +1666 Ripe (1996) +1667 Next Step, The (1995) +1668 Wedding Bell Blues (1996) +1669 MURDER and murder (1996) +1670 Tainted (1998) +1671 Further Gesture, A (1996) +1672 Kika (1993) +1673 Mirage (1995) +1674 Mamma Roma (1962) +1675 Sunchaser, The (1996) +1676 War at Home, The (1996) +1677 Sweet Nothing (1995) +1678 Mat' i syn (1997) +1679 B. Monkey (1998) +1680 Sliding Doors (1998) +1681 You So Crazy (1994) +1682 Scream of Stone (Schrei aus Stein) (1991) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6f9d8527/s2core/src/test/resources/user_factors.ann ---------------------------------------------------------------------- diff --git a/s2core/src/test/resources/user_factors.ann b/s2core/src/test/resources/user_factors.ann new file mode 100644 index 0000000..ad3c663 Binary files /dev/null and b/s2core/src/test/resources/user_factors.ann differ http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6f9d8527/s2core/src/test/scala/org/apache/s2graph/core/model/FetcherTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/model/FetcherTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/model/FetcherTest.scala index fb6ff16..54e6763 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/model/FetcherTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/model/FetcherTest.scala @@ -135,7 +135,7 @@ class FetcherTest extends IntegrateCommon{ val stepResult = Await.result(graph.getEdges(query), Duration("60 seconds")) stepResult.edgeWithScores.foreach { es => - println(es.edge) + println(es.edge.tgtVertex.innerIdVal) } FileUtils.deleteQuietly(new File(localIndexFilePath)) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6f9d8527/s2jobs/build.sbt ---------------------------------------------------------------------- diff --git a/s2jobs/build.sbt b/s2jobs/build.sbt index e966fa6..1430500 100644 --- a/s2jobs/build.sbt +++ b/s2jobs/build.sbt @@ -32,12 +32,15 @@ libraryDependencies ++= Seq( "org.apache.spark" %% "spark-core" % spark2Version, "org.apache.spark" %% "spark-streaming" % spark2Version % "provided", "org.apache.spark" %% "spark-hive" % spark2Version % "provided", + "org.apache.spark" %% "spark-mllib" % spark2Version, "org.specs2" %% "specs2-core" % specs2Version % "test", "org.scalatest" %% "scalatest" % "2.2.1" % "test", "org.apache.hadoop" % "hadoop-distcp" % hadoopVersion, "org.elasticsearch" % "elasticsearch-spark-20_2.11" % elastic4sVersion, "com.github.scopt" %% "scopt" % "3.7.0", - "com.holdenkarau" %% "spark-testing-base" % "2.3.0_0.9.0" % Test + "com.holdenkarau" %% "spark-testing-base" % "2.3.0_0.9.0" % Test, + "net.pishen" %% "annoy4s" % annoy4sVersion, + "org.tensorflow" % "tensorflow" % tensorflowVersion ) crossScalaVersions := Seq("2.10.6") http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6f9d8527/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/custom/process/ALSAnnoyBuildProcess.scala ---------------------------------------------------------------------- diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/custom/process/ALSAnnoyBuildProcess.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/custom/process/ALSAnnoyBuildProcess.scala new file mode 100644 index 0000000..968bf5a --- /dev/null +++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/task/custom/process/ALSAnnoyBuildProcess.scala @@ -0,0 +1,87 @@ +package org.apache.s2graph.s2jobs.task.custom.process + +import java.io.File + +import annoy4s.{Angular, Annoy} +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileSystem, FileUtil, Path} +import org.apache.s2graph.s2jobs.task.TaskConf +import org.apache.spark.ml.recommendation.{ALS, ALSModel} +import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession} + +object ALSAnnoyBuildProcess { + + def buildAnnoyIndex(ss: SparkSession, + conf: TaskConf, + dataFrame: DataFrame): Unit = { + // annoy tree params. + val outputPath = conf.options("outputPath") + val localInputPath = conf.options("localInputPath") + val localIndexPath = conf.options("localIndexPath") + val numDimensions = conf.options.getOrElse("dimensions", "10").toInt + + // als model params. + val rank = conf.options.getOrElse("rank", numDimensions.toString).toInt + val maxIter = conf.options.getOrElse("maxIter", "5").toInt + val regParam = conf.options.getOrElse("regParam", "0.01").toDouble + val userCol = conf.options.getOrElse("userCol", "userId") + val itemCol = conf.options.getOrElse("itemCol", "movieId") + val ratingCol = conf.options.getOrElse("ratingCol", "rating") + + assert(rank == numDimensions) + + val als = new ALS() + .setRank(rank) + .setMaxIter(maxIter) + .setRegParam(regParam) + .setUserCol(userCol) + .setItemCol(itemCol) + .setRatingCol(ratingCol) + + val model = als.fit(dataFrame) + + saveFeatures(ss, model.itemFactors, outputPath) + copyToLocal(ss.sparkContext.hadoopConfiguration, outputPath, localInputPath) + + FileUtil.fullyDelete(new File(localIndexPath)) + + Annoy.create[Int](s"${localInputPath}", numDimensions, outputDir = s"$localIndexPath", Angular) + } + + def saveFeatures(ss: SparkSession, + dataFrame: DataFrame, + outputPath: String, + idCol: String = "id", + featuresCol: String = "features"): Unit = { + import ss.sqlContext.implicits._ + + val result = dataFrame.map { row => + val id = row.getAs[Int](idCol) + val vector = row.getAs[Seq[Float]](featuresCol) + (Seq(id) ++ vector).mkString(" ") + } + + result.write.mode(SaveMode.Overwrite).csv(outputPath) + } + + def copyToLocal(configuration: Configuration, + remoteInputPath: String, + localOutputPath: String, + merge: Boolean = true): Unit = { + val fs = FileSystem.get(configuration) + val localFs = FileSystem.getLocal(configuration) + localFs.deleteOnExit(new Path(localOutputPath)) + + if (merge) + FileUtil.copyMerge(fs, new Path(remoteInputPath), localFs, new Path(localOutputPath), false, configuration, "") + else + fs.copyToLocalFile(new Path(remoteInputPath), new Path(localOutputPath)) + } +} +class ALSAnnoyBuildProcess(conf: TaskConf) extends org.apache.s2graph.s2jobs.task.Process(conf) { + override def execute(ss: SparkSession, inputMap: Map[String, DataFrame]): DataFrame = ??? + + override def mandatoryOptions: Set[String] = Set("outputPath") + + +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6f9d8527/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/task/custom/process/ALSAnnoyBuildProcessTest.scala ---------------------------------------------------------------------- diff --git a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/task/custom/process/ALSAnnoyBuildProcessTest.scala b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/task/custom/process/ALSAnnoyBuildProcessTest.scala new file mode 100644 index 0000000..6279e82 --- /dev/null +++ b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/task/custom/process/ALSAnnoyBuildProcessTest.scala @@ -0,0 +1,104 @@ +package org.apache.s2graph.s2jobs.task.custom.process + +import java.io.File + +import com.holdenkarau.spark.testing.DataFrameSuiteBase +import com.typesafe.config.ConfigFactory +import org.apache.commons.io.FileUtils +import org.apache.s2graph.core.Integrate.IntegrateCommon +import org.apache.s2graph.core.Management.JsonModel.{Index, Prop} +import org.apache.s2graph.core.{Query, QueryParam} +import org.apache.s2graph.core.model.{AnnoyModelFetcher, HDFSImporter, ModelManager} +import org.apache.s2graph.core.schema.Label +import org.apache.s2graph.s2jobs.task.TaskConf + +import scala.concurrent.{Await, ExecutionContext} +import scala.concurrent.duration.Duration +import scala.io.Source + +class ALSAnnoyBuildProcessTest extends IntegrateCommon with DataFrameSuiteBase { + import scala.collection.JavaConverters._ + + // this test require adding movie lens rating data(u.data, movie.txt) under resources + // so ignore for now until figure out how to automate download dataset. + ignore("RUN ALS on movie lens rating data and build annoy index on itemFeatures, finally query.") { + import spark.sqlContext.implicits._ + val ratingPath = this.getClass.getResource("/u.data").toURI.getPath + + val ratings = Source.fromFile(new File(ratingPath)).getLines().toSeq.map { line => + val tokens = line.split("\t") + (tokens(0).toInt, tokens(1).toInt, tokens(2).toFloat) + }.toDF("userId", "movieId", "rating") + + val outputPath = "/tmp" + val localInputPath = "/tmp/annoy_input" + val localIndexPath = "/tmp/annoy_result" + + val taskOptions = Map( + "outputPath" -> outputPath, + "localInputPath" -> localInputPath, + "localIndexPath" -> localIndexPath + ) + + val conf = TaskConf("test", "test", Nil, taskOptions) + ALSAnnoyBuildProcess.buildAnnoyIndex(spark, conf, ratings) + + val labelName = "annoy_model_fetcher_test" + + val remoteIndexFilePath = s"${localIndexPath}/annoy-index" + val remoteDictFilePath = this.getClass.getResource(s"/movie.dict").toURI.getPath + + val service = management.createService("s2graph", "localhost", "s2graph_htable", -1, None).get + val serviceColumn = + management.createServiceColumn("s2graph", "user", "string", Seq(Prop("age", "0", "int", true))) + + val options = s"""{ + | "importer": { + | "${ModelManager.ImporterClassNameKey}": "org.apache.s2graph.core.model.IdentityImporter" + | }, + | "fetcher": { + | "${ModelManager.FetcherClassNameKey}": "org.apache.s2graph.core.model.AnnoyModelFetcher", + | "${AnnoyModelFetcher.IndexFilePathKey}": "${remoteIndexFilePath}", + | "${AnnoyModelFetcher.DictFilePathKey}": "${remoteDictFilePath}", + | "${AnnoyModelFetcher.DimensionKey}": 10 + | } + |}""".stripMargin + + Label.findByName(labelName, useCache = false).foreach { label => Label.delete(label.id.get) } + + val label = management.createLabel( + labelName, + serviceColumn, + serviceColumn, + true, + service.serviceName, + Seq.empty[Index].asJava, + Seq.empty[Prop].asJava, + "strong", + null, + -1, + "v3", + "gz", + options + ) + val config = ConfigFactory.parseString(options) + val importerFuture = graph.modelManager.importModel(label, config)(ExecutionContext.Implicits.global) + Await.result(importerFuture, Duration("3 minutes")) + + Thread.sleep(10000) + + val vertex = graph.elementBuilder.toVertex(service.serviceName, serviceColumn.columnName, "Toy Story (1995)") + val queryParam = QueryParam(labelName = labelName, limit = 5) + + val query = Query.toQuery(srcVertices = Seq(vertex), queryParams = Seq(queryParam)) + val stepResult = Await.result(graph.getEdges(query), Duration("60 seconds")) + + stepResult.edgeWithScores.foreach { es => + println(es.edge.tgtVertex.innerIdVal) + } + + // clean up temp directory. + FileUtils.deleteDirectory(new File(outputPath)) + } + +}
