This is the program:
;;;;
;;;; Playing with Netflix dataset using Elephant to store data
;;;;
(mapc (lambda (x) (asdf:oos 'asdf:load-op x))
'(:elephant :iterate :split-sequence))
(defpackage :nelephant
(:use :cl
:elephant
:iterate
:split-sequence
))
(in-package :nelephant)
(deftype int32 () '(integer 32))
(defconstant +movies-num+ 17770 "Total number of movies")
(open-store '(:BDB "/home/victor/netflix/db/"))
;(open-store '(:CLSQL (:SQLITE3 "/home/victor/netflix/sqldb.db")))
(defpclass ratings-collection ()
((id :type int32 :initarg :id :reader get-id :index t
:documentation "Object id")
(ratings :initform (make-btree) :reader ratings
:documentation "A hash mapping ids to ratings"))
(:documentation "An abstract ratings collection"))
(defpclass user (ratings-collection)
()
(:documentation "Movies rated by user with this ID"))
(defpclass movie (ratings-collection)
()
(:documentation "Users rated movie with this ID"))
(defun get-or-create-by-id (id object)
"Return existing object with given ID or create a new one if it
doesn't exists"
(or (first (get-instances-by-value object 'id id))
(make-instance object :id id)))
(declaim (inline get-user))
(defun get-user (id)
"Return user object with given ID or create a new one"
(get-or-create-by-id id 'user))
(declaim (inline get-movie))
(defun get-movie (id)
"Return movie object with given ID or create a new one"
(get-or-create-by-id id 'movie))
(defun movie-filename (n)
"Data file with ratings on film N"
(format nil "download/training_set/mv_~7,'0D.txt" n))
(defun collect-rating-info (movie line)
"Parse rating data for MOVIE from LINE"
(destructuring-bind (user rating date)
(split-sequence #\, line)
(declare (ignore date))
(setf (get-value movie (ratings (get-user user)))
rating
(get-value user (ratings (get-movie movie)))
rating)))
(defun import-movie (n)
"Import ratings information for movie number N."
(with-transaction ()
(iter (for line in-file (movie-filename n) using 'read-line)
(unless (first-time-p) ; skip first line
(collect-rating-info n line)))))
(defun fprinc (obj)
(princ obj) (princ #\Space)
(sb-int:flush-standard-output-streams))
(defun import-all-movies ()
"Import all netflix movies ratings from text files."
(iter (for i from 1 to +movies-num+)
(import-movie i)
(fprinc i)))
On Jan 4, 2008 1:12 AM, Victor Kryukov <[EMAIL PROTECTED]> wrote:
> Hello list,
>
> I've decided to put elephant 0.9.1 under some heavy load test, and
> play with Netflix data set a little bit. The attached program that
> tries to import everything in BerkeleyDB fails when trying to import
> movie file number 8 with the following traceback. Do you have any idea
> what could be the problem?
>
> Thanks,
> Victor.
>
> Berkeley DB error: Cannot allocate memory
> [Condition of type BDB-DB-ERROR]
>
> Restarts:
> 0: [ABORT-REQUEST] Abort handling SLIME request.
> 1: [ABORT] Exit debugger, returning to top level.
>
> Backtrace:
> 0: ((SB-PCL::FAST-METHOD (SETF GET-VALUE) (T T DB-BDB::BDB-BTREE))
> #<unavailable lambda list>)
> 1: ((LAMBDA NIL))
> 2: ((SB-PCL::FAST-METHOD ELEPHANT::EXECUTE-TRANSACTION
> (DB-BDB::BDB-STORE-CONTROLLER #1="#<...>" . #1#)) #<unavailable
> argument> #<unavailable argument> #<BDB-STORE-CONTROLLER
> /home/victor/netflix/db/> #<CLOSURE (LAMBDA NIL) {1003DC2359}>)
> 3: (IMPORT-ALL-MOVIES)
> 4: (SB-INT:SIMPLE-EVAL-IN-LEXENV (IMPORT-ALL-MOVIES) #<NULL-LEXENV>)
> 5: (SWANK::EVAL-REGION "(import-all-movies)
> " T)
> 6: ((LAMBDA NIL))
> 7: ((SB-PCL::FAST-METHOD SWANK-BACKEND:CALL-WITH-SYNTAX-HOOKS (T))
> #<unused argument> #<unused argument> #<CLOSURE (LAMBDA NIL)
> {100403F959}>)
> 8: (SWANK::CALL-WITH-BUFFER-SYNTAX #<CLOSURE (LAMBDA NIL) {100403F959}>)
> 9: (SB-INT:SIMPLE-EVAL-IN-LEXENV (SWANK:LISTENER-EVAL "(import-all-movies)
> ") #<NULL-LEXENV>)
> 10: ((LAMBDA NIL))
> 11: ((SB-PCL::FAST-METHOD SWANK-BACKEND:CALL-WITH-DEBUGGER-HOOK (T
> T)) #<unused argument> #<unused argument> #<FUNCTION
> SWANK:SWANK-DEBUGGER-HOOK> #<CLOSURE (LAMBDA NIL) {100403F049}>)
> 12: ((LAMBDA NIL))
> 13: ((SB-PCL::FAST-METHOD SWANK-BACKEND:CALL-WITH-DEBUGGER-HOOK (T
> T)) #<unused argument> #<unused argument> #<FUNCTION
> SWANK:SWANK-DEBUGGER-HOOK> #<FUNCTION (LAMBDA NIL) {10028DAF69}>)
> 14: (SWANK::CALL-WITH-REDIRECTED-IO #<SWANK::CONNECTION {1002EFB1E1}>
> #<CLOSURE (LAMBDA NIL) {100403C2F9}>)
> 15: (SWANK::CALL-WITH-CONNECTION #<SWANK::CONNECTION {1002EFB1E1}>
> #<FUNCTION (LAMBDA NIL) {10028DAF69}>)
> 16: (SWANK::HANDLE-REQUEST #<SWANK::CONNECTION {1002EFB1E1}>)
> 17: (SWANK::PROCESS-AVAILABLE-INPUT #<SB-SYS:FD-STREAM for "a socket"
> {1002EE6A01}> #<CLOSURE (LAMBDA NIL) {100403C279}>)
> 18: ((FLET SWANK::HANDLER))
> 19: ((LAMBDA (SWANK-BACKEND::_)) #<unused argument>)
> 20: (SB-IMPL::SUB-SERVE-EVENT NIL NIL NIL)
> 21: (SB-SYS:WAIT-UNTIL-FD-USABLE 0 :INPUT NIL)
> 22: (SB-IMPL::REFILL-INPUT-BUFFER #<SB-SYS:FD-STREAM for "standard
> input" {1002393801}>)
> 23: (SB-IMPL::INPUT-CHAR/UTF-8 #<SB-SYS:FD-STREAM for "standard
> input" {1002393801}> NIL #:EOF-OBJECT)
> 24: (READ-CHAR #<SB-SYS:FD-STREAM for "standard input" {1002393801}>
> NIL #:EOF-OBJECT #<unused argument>)
> 25: (READ-CHAR #<SYNONYM-STREAM :SYMBOL SB-SYS:*STDIN* {100014AEB1}>
> NIL #:EOF-OBJECT #<unused argument>)
> 26: (READ-CHAR #<SYNONYM-STREAM :SYMBOL
> SWANK::*CURRENT-STANDARD-INPUT* {1002384001}> NIL #:EOF-OBJECT
> #<unused argument>)
> 27: (READ-PRESERVING-WHITESPACE #<SYNONYM-STREAM :SYMBOL
> SWANK::*CURRENT-STANDARD-INPUT* {1002384001}> NIL (NIL) T)
> 28: (READ-PRESERVING-WHITESPACE #<SYNONYM-STREAM :SYMBOL
> SWANK::*CURRENT-STANDARD-INPUT* {1002384001}> NIL (NIL) NIL)
> 29: (READ #<SYNONYM-STREAM :SYMBOL SWANK::*CURRENT-STANDARD-INPUT*
> {1002384001}> NIL (NIL) NIL)
> 30: (SB-IMPL::REPL-READ-FORM-FUN #<SYNONYM-STREAM :SYMBOL
> SWANK::*CURRENT-STANDARD-INPUT* {1002384001}> #<unavailable argument>)
> 31: (SB-IMPL::REPL-FUN NIL)
> 32: (SB-IMPL::REPL-FUN NIL)
> 33: ((LAMBDA NIL))
> 34: ((LAMBDA NIL))
> 35: (SB-IMPL::%WITH-REBOUND-IO-SYNTAX #<CLOSURE (LAMBDA NIL) {100386A4D9}>)
> 36: (SB-IMPL::TOPLEVEL-REPL NIL)
> 37: (SB-IMPL::TOPLEVEL-INIT)
> 38: ((LABELS SB-IMPL::RESTART-LISP))
>
>
> --
> Yours Sincerely,
> Victor Kryukov
> Chicago GSB class of 2008
> 773-618-9501
>
--
Yours Sincerely,
Victor Kryukov
Chicago GSB class of 2008
773-618-9501
_______________________________________________
elephant-devel site list
elephant-devel@common-lisp.net
http://common-lisp.net/mailman/listinfo/elephant-devel
