/*
	ZillionServer.m
	Tue Aug 27 17:02:16 CEST 2002
 */

#import	"ZillionServer.h"
#import	"NSProcessInfo+getopt.h"
#import	"NSFileManager+directories.h"

#define	ZILLION_LAST_SEEDID_KEY	@"lastSeedId"

/* keys of a job specification */
#define	PACKED_BUNDLE_KEY	@"packedBundle"
#define	JOB_INSTANCE_KEY	@"jobInstance"
#define	SEEDSToBeRESCHEDULED_KEY	@"rescheduledSeeds"

/*
	Outline of the protocol:
	The server starts up and offers up his services under the name ZILLION_SERVER.
	A client can register via DO with the server to offer calculation power.
	(-registerClient:)
		- the client has properties:
			- load: how many jobs are submitted in parallel
			- memory: how much memory is available
			- ... to be extended

	A Job is then initiated as follows:
	- The server receives a -dispatchJob: method. This takes a NSData object representing
		a bundle which encodes the job. A job Id is returned.

	- The job is then initiated by choosing from the clients and sending a -loadJob:
		method forwarding the bundle from the server
	- The server then asks the job for a seedEnumerator and for a seed
	- This seed is transferred to the client via -dispatchUnitFromSeed:forJobId:
	- which in turn detaches a new thread and asks the job to -startJobWithClient:andSeed:
	- results are returned 

 */

#define	PROXYSERVER_FOR_JOBID(jobId) (!_respawnStatus? self: [self serverForJobId:jobId])

#define	RESCHEDULE()	[self reschedule]

#define	SEALJOBWITHID(jobId) do { \
	[[NSRunLoop currentRunLoop] performSelector:@selector(sealJobWithId:) \
		target:self \
		argument:[jobId copy] order:1 \
		modes:[NSArray arrayWithObject:NSDefaultRunLoopMode]]; \
	NSLog(@"scheduled sealing of job:%@", jobId);\
	} while (0)

#define	JobSeedStatus(j) \
	(!![[self jobSpecificationForJobId:j] objectForKey:@"seedsAreExhausted"])
#define	SetJobSeedsExhausted(j) \
	[[self jobSpecificationForJobId:j] setObject:@"YES" forKey:@"seedsAreExhausted"]


@implementation ZillionServer

/*
	The idea for respawning a new server is that classes cannot
	be unloaded in objc currently due to reentrance problems which would occur otherwise.

	The new server is started and 
 */

- (NSString *)DOname { return ZILLION_SERVER_NAME; }
- (NSArray *)respawningArgumentsWithId:(NSString *)doNameId {
	return [NSArray arrayWithObjects:@"--serverName", [self respawningDOnameWithId:doNameId],
		@"--doRespawn=NO", nil];
}
- (Protocol *)respawningProtocol { return @protocol(ZillionServerProtocol); }

- initWithRespawnStatus:(BOOL)respawnStatus
{
	if (!(self = [super init])) return self;

	clients = [[NSMutableDictionary alloc] init];
	_respawnStatus = respawnStatus;
	proxyServers = [[NSMutableDictionary alloc] init];

	return self;
}
- init
{
	return [self initWithRespawnStatus:NO];
}

- (void)dealloc
{
	NSLog(@"about to dealloc ZillionServer object");

	[clients release];
	[proxyServers release];
	[super dealloc];
}

- (void)reschedule
{
	if (_reschedulingIsMarked) return;
	_reschedulingIsMarked = YES;
	[[NSRunLoop currentRunLoop] performSelector:@selector(scheduleJobs:)
		target:self argument:nil order:0 modes:[NSArray arrayWithObject:NSDefaultRunLoopMode]];
}


- (void)clientConnectionDidDie:(NSNotification *)notification
{
	// presume this is the client <t>
	[self deregisterClient:[[notification object] rootObject]];
}

- (void)registerClient:(id <ZillionClientProtocol>)someClient
{
	NSDictionary	*newClient = [NSDictionary dictionaryWithObjectsAndKeys:
		someClient, @"object", [someClient attributes], @"attributes", nil];

	[clients setObject:newClient forKey:[someClient clientId]];

	// <!> see doc for possible drawbacks; we might not notice invalidation
	// unless we ping the client <i>
	[[NSNotificationCenter defaultCenter] addObserver:self
		selector:@selector(clientConnectionDidDie:)
		name:NSConnectionDidDieNotification
		object:[(NSDistantObject *)someClient connectionForProxy]];

	NSLog(@"Registered client:%@", someClient);
	RESCHEDULE();
}

- (void)deregisterClient:(id <ZillionClientProtocol>)someClient
{
	[clients removeObjectForKey:[someClient clientId]];
	RESCHEDULE();
}

- (id <ZillionServerProtocol>)serverForJobId:(NSString *)jobId {
	return [proxyServers objectForKey:jobId];
}


- (void)respawnedConnectionDidDie:(NSNotification *)notification
{
	NSLog(@"Some proxy went away");
}

- (NSString *)dispatchJob:(bycopy NSData *)bundle withResources:(NSData *)resources
{
	id <ZillionServerProtocol>	proxyServer = self;
	NSString	*jobId;

	//	we respawn a server to loadd the job 
	if (_respawnStatus) {
		proxyServer = [self respawnedObjectWithId:
			[[NSNumber numberWithInt:proxyServerIdCounter] stringValue]
			doObserveConnection:YES];
		[proxyServer setMasterServer:self];
		proxyServerIdCounter++;
	}

	jobId = [proxyServer loadAndRegisterJob:bundle];

	if (_respawnStatus) {
		[proxyServers setObject:proxyServer forKey:jobId];
	}
	RESCHEDULE();
	return jobId;
}

- (NSString *)dispatchJob:(NSData *)bundle
{
	return [self dispatchJob:bundle withResources:nil];
}

- (NSString *)description { NSLog(@"description forthcoming");
	return [super description];
}

//	called in the proxyServer

- (void)sealJobWithId:(NSString *)jobId
{
	NSLog(@"Welcome to the finished job [%d]:%@", _respawnStatus, jobId);
	[[self jobInstance:jobId] sealJobOutcomes];

	if (_respawnStatus) {
		id <ZillionServerProtocol>	proxy = PROXYSERVER_FOR_JOBID(jobId);
		NSLog(@"did seal job in proxy for job:%@, now terminating", jobId);
		[proxy terminate];
		[proxyServers removeObjectForKey:jobId];
	}
}

- (BOOL)serverToGoDown { return _serverToGoDown; }
- (void)terminate { 

	NSLog(@"Server is about to terminate");
	if (_respawnStatus) {
		NSEnumerator				*proxies = [proxyServers objectEnumerator];
		id <ZillionServerProtocol>	proxyServer;
		while ( (proxyServer = [proxies nextObject]) )
			[proxyServer terminate];
	}
	if (!_respawnStatus) {
		// handle running jobs gracefully <i>
	}

	_serverToGoDown = YES;
}

- (void)registerJob:(NSData *)jobBundle
	withId:(NSString *)jobId
	resources:(NSData *)resources
	instance:(id <ZillionJobProtocol>)jobInstance
{
	NSMutableDictionary	*jobSpec = [self jobSpecificationForJobId:jobId];

	NSLog(@"Registering %@.", jobId);
	[jobSpec setObject:jobBundle forKey:PACKED_BUNDLE_KEY];
	[jobSpec setObject:[NSNumber numberWithDouble:0] forKey:@"timePenalty"];
	if (resources) [jobSpec setObject:[resources copy] forKey:JOB_RESOURCES_KEY];
	[jobSpec setObject:jobInstance forKey:JOB_INSTANCE_KEY];
	NSLog(@"Did register job %@.", jobId);
}

- (NSData *)packedJobBundleForJobId:(NSString *)jobId
{
	return [[self jobSpecificationForJobId:jobId] objectForKey:PACKED_BUNDLE_KEY];
}

- (id <ZillionJobProtocol>)loadJob:(NSData *)packedBundle withResources:(NSData *)jobResources
{
	NSBundle		*jobBundle = [self bundleForPackedBundle:packedBundle];
	id <ZillionJobProtocol>
					jobInstance = [self instantiateJobFromBundle:jobBundle
						withResources:jobResources];

	NSLog(@"...loaded the code for job %@.", [jobInstance jobId]);
	return jobInstance;
}

- (NSString *)loadAndRegisterJob:(NSData *)packedBundle withResources:(NSData *)jobResources
{
	id <ZillionJobProtocol> jobInstance = [self loadJob:packedBundle withResources:jobResources];
	NSString		*jobId = [jobInstance jobId];

	[[self masterServer] registerJob:packedBundle
		withId:jobId
		resources:jobResources
		instance:jobInstance	// on the master server a proxy shall reside
	];
	return jobId;
}

- (NSString *)loadAndRegisterJob:(NSData *)packedBundle
{
	return [self loadAndRegisterJob:packedBundle withResources:nil];
}

/*
	this returns a unique jobInstance
	which is created by loadAndRegisterJob:withResources:
 */
- (id <ZillionJobProtocol>)jobInstance:(NSString *)jobId
{
	NSMutableDictionary	*jobSpec = [[self masterServer] jobSpecificationForJobId:jobId];

	return [jobSpec objectForKey:JOB_INSTANCE_KEY];
}

- (NSData *)nextSeedIdForJobId:(NSString *)jobId
{
	// trigger job instantiation
	id <ZillionJobProtocol>	job = [self jobInstance:jobId];
	NSMutableDictionary		*jobSpec = [self jobSpecificationForJobId:jobId];
	NSEnumerator			*seedEnum = [jobSpec objectForKey:@"enumerator"];
	NSData					*seedId;

	if (!seedEnum) {
		NSLog(@"new seed enum");
		seedEnum = [job seedEnumeratorWithSeed:nil];
		[jobSpec setObject:seedEnum forKey:@"enumerator"];
	}

	seedId = [job dataRepresentationOfSeed:[seedEnum nextObject]];
	if (seedId) {
		[jobSpec setObject:seedId forKey:ZILLION_LAST_SEEDID_KEY];
	} else {
		[jobSpec removeObjectForKey:ZILLION_LAST_SEEDID_KEY];
	}
	return seedId;
}

#define	JOBPenalty(j) \
	[[[self jobSpecificationForJobId:j] objectForKey:@"timePenalty"] doubleValue]
#define	JOBNextSeed(j) \
	[[[self jobSpecificationForJobId:j] objectForKey:@"seedEnumerator"] nextObject]
#define	JOBIncPenatly(j, p)	\
	do { \
		NSMutableDictionary	*jDict = [self jobSpecificationForJobId:j]; \
		[jDict setObject:[NSNumber numberWithDouble:(p) + \
			[[jDict objectForKey:@"timePenalty"] doubleValue]] \
			forKey:@"timePenalty"]; \
	} while(0)

#define	ZillionOutcomesKey	@"outcomes"

#if 0
- (NSMutableDictionary *)jobOutcomesForJobId:(NSString *)jobId
{
	NSMutableDictionary	*jobOutcomes = [outcomes objectForKey:jobId];

	if (!jobOutcomes) {
		jobOutcomes = [NSMutableDictionary dictionary];
		[outcomes setObject:jobOutcomes forKey:jobId];
		[jobOutcomes setObject:[NSMutableArray array] forKey:@"finishedSeeds"];
		[jobOutcomes setObject:[NSMutableDictionary dictionary] forKey:ZillionOutcomesKey];
	}
	return jobOutcomes;
}
#endif

- (void)submitOutcome:(NSData *)outcome ofJob:(NSString *)jobId forSeedId:(NSData *)seedId
	storeHints:(NSDictionary *)someStoreHints
{
	id <ZillionJobProtocol, ZillionJobOutcomeProcessing>	job = [self jobInstance:jobId];

	NSLog(@"saving in the server");
#if	0
	if ([job respondsToSelector:@selector(saveOutcome:forSeedId:storeHints:)]) {
		[job saveOutcome:outcome forSeedId:seedId storeHints:someStoreHints];
	}
#endif
}


/*
	seeds should be allowed to go without descriptions <i>
 */

- (void)writeOutcomeToDisk:(bycopy NSData *)outcome forSeedId:(bycopy NSData *)seedId
	andJobId:(bycopy NSString *)jobId
{
	id <ZillionJobProtocol>	job = [self jobInstance:jobId];
	id <ZillionSeed>		seed = [job seedObjectFromDataRepresentation:seedId];
	NSString				*jobPath = [[self serverTemporaryPath]
		stringByAppendingPathComponents:@"Scratch", jobId, [seed description], nil];

	NSMutableDictionary	*jobOutcomes = [[self masterServer] outcomeDictionaryForJobId:jobId];

	[[jobOutcomes objectForKey:ZillionOutcomesKey] setObject:jobPath forKey:seedId];
	[outcome writeToFile:jobPath atomically:NO];
}

/*
	we'll store outcomes if we are asked to do so
 */

- (void)setOutcome:(bycopy NSData *)outcome forSeedId:(bycopy NSData *)seedId
	andJobId:(bycopy NSString *)jobId
{
	NSMutableDictionary	*jobOutcomes = [self outcomeDictionaryForJobId:jobId];
	id <ZillionJobProtocol>	job = [self jobInstance:jobId];

	if ([job storesOutcomesInMemory]) {
		[[jobOutcomes objectForKey:ZillionOutcomesKey] setObject:outcome forKey:seedId];
	} else {
		[PROXYSERVER_FOR_JOBID(jobId) writeOutcomeToDisk:outcome
			forSeedId:seedId andJobId:jobId];
	}
}

- (NSMutableDictionary *)outcomeDictionaryForJobId:(NSString *)jobId
{
	NSMutableDictionary	*jobSpecification = [self jobSpecificationForJobId:jobId];
	NSMutableDictionary	*jobOutcomes = [jobSpecification objectForKey:@"outcomes"];

	if (!jobOutcomes) {
		jobOutcomes = [NSMutableDictionary dictionary];
		[jobSpecification setObject:jobOutcomes forKey:@"outcomes"];
	}
	return jobOutcomes;

	return [[self outcomeDictionaryForJobId:jobId] objectForKey:ZillionOutcomesKey];
}

- (void)setOutcomeDictionary:(bycopy NSMutableDictionary *)outcomes forJobId:(bycopy NSString *)jobId
{
	NSMutableDictionary	*jobSpecification = [self jobSpecificationForJobId:jobId];

	[jobSpecification setObject:outcomes forKey:@"outcomes"];
}


- (NSMutableSet *)dispatchedSeedsForJobId:(NSString *)jobId
{
	NSMutableDictionary	*jobSpecification = [self jobSpecificationForJobId:jobId];
	NSMutableSet		*seeds = [jobSpecification objectForKey:@"dispatchedSeeds"];

	if (!seeds) {
		seeds = [NSMutableSet set];
		[jobSpecification setObject:seeds forKey:@"dispatchedSeeds"];
	}
	return seeds;
}

- (void)clearDispatchedSeedsSetForJobId:(NSString *)jobId
{
	NSMutableDictionary	*jobSpecification = [self jobSpecificationForJobId:jobId];
	NSMutableSet		*seeds = [jobSpecification objectForKey:@"dispatchedSeeds"];

	[seeds removeAllObjects];
}

- (oneway void)jobUnitDidEndForJobId:(bycopy NSString *)jobId seedId:(bycopy NSData *)seedId
	timePenalty:(NSTimeInterval)penalty
{
	NSMutableSet		*seeds = [self dispatchedSeedsForJobId:jobId];

	NSLog(@"job unit is about to be ended in the server");
	JOBIncPenatly(jobId, penalty);

	if (![seeds containsObject:seedId]) {
		[NSException raise:NSInternalInconsistencyException
			format:@"Finished Seed was not registered [%@]", seedId];
	} else {
		[seeds removeObject:seedId];
	}

	// are we ready to seal the job?
	if (![seeds count] && JobSeedStatus(jobId))
		SEALJOBWITHID(jobId);
	RESCHEDULE();
	NSLog(@"job unit was ended in the server");
}

- (void)registerSeed:(bycopy NSData *)seedForJob forJob:(NSString *)jobId
{
	[[self dispatchedSeedsForJobId:jobId] addObject:seedForJob];
}


#if 0
- (double)cumulatedPenalty
{
	NSEnumerator	*jobEnum = [outcomes keyEnumerator];
	NSDictionary	*job;
	double	penalty = 0;

	while (job = [jobEnum nextObject]) {	
		penalty += JOBPenalty(job);
	}
	return penalty;
}
#endif

// this is a biggest residual seat distribution scheme where
//	dividend is the count of seats
//	relative is an array of (relative) votes
//	arrayLength is the number of parties and
//	result holds the number of seats after the distribution of seats

typedef struct {
	int		index;
	double	residual;
} Residual;
int cmpResidual(Residual *r1, Residual *r2) {
	return r1->residual == r2->residual? 0
		: (r1->residual < r2->residual? -1: 1);
}
typedef int (*QSortComparator)(const void*, const void*);

static void	scaleSetTo(double *relative, unsigned arrayLength, unsigned dividend, unsigned *result)
{	unsigned	i, cSum, cRest;
	double		rSum;
	Residual	residuals[arrayLength];

	for (i = 0, rSum = 0; i < arrayLength; i++) rSum += relative[i], result[i] = 0;
	if (!rSum) return;

	for (i = 0; i < arrayLength; i++)
		result[i] = (int)(relative[i] / rSum * arrayLength);

	do {
		for (i = 0; i < arrayLength; i++) {
			residuals[i] = ((Residual){i, result[i] / rSum * arrayLength - result[i]});
		}
		qsort(residuals, arrayLength, sizeof(Residual), (QSortComparator)cmpResidual);
		for (i = cSum = 0; i < arrayLength; i++) cSum += result[i];
		for (i = 0, cRest = dividend - cSum; i < arrayLength && cRest; i++, cRest--)
			result[residuals[i].index]++;
	} while (cRest);
}

// <i> reprioritize when a new job is started or an old one ends

#define	CLIENT_MAX_LOAD(c)	[[[c objectForKey:@"attributes"] objectForKey:@"load"] intValue]

- (void)scheduleJobs:sender
{
	NSEnumerator	*clientEnum;
	NSString		*clientId;
	NSMutableArray	*freeClients = [NSMutableArray array];

	NSLog(@"Scheduling jobs...");
	// clear flag
	_reschedulingIsMarked = NO;

	// determine which clients to run on
	for (clientEnum = [clients keyEnumerator]; (clientId = [clientEnum nextObject]); ) {
		NSDictionary	*clientSpec = [clients objectForKey:clientId];
		id <ZillionClientProtocol>	client = [clientSpec objectForKey:@"object"];
		int		i, count;
		int		currentLoad = [client currentLoad];
		for (i = 0, count = CLIENT_MAX_LOAD(clientSpec) - currentLoad ; i < count; i++)
			[freeClients addObject:client];
	}
NSLog(@"Free client count %d", [freeClients count]);
	if (![freeClients count]) return;

	NS_DURING
	// determine which jobs to run
	//	the inverse relative time penatly is proportional to the share in clients for each job
	{
		NSArray			*jobIds = [self jobIds];
		int				jobCount = [jobIds count];
		double			penalties[jobCount], penalty;
		NSString		*jobId;
		int				i, j, k, count, slots[jobCount];
		id <ZillionServerProtocol>	proxyServer;

		for (i = 0, count = [jobIds count]; i < count; i++) {
			jobId = [jobIds objectAtIndex:i];
			penalty = JOBPenalty(jobId);
			//<i> rather than setting to 1, set to priority of the job in the instance
			penalties[i] = penalty? 1/penalty: 1;
		}
		scaleSetTo(penalties, jobCount, [freeClients count], slots);

		for (i = k = 0, count = [jobIds count]; i < count; i++) {	
			jobId = [jobIds objectAtIndex:i];
			for (j = 0; j < slots[i]; j++) {
				NSData	*seedForJob;
				proxyServer = PROXYSERVER_FOR_JOBID(jobId);
				seedForJob = [proxyServer nextSeedIdForJobId:jobId];
				if (!seedForJob) {
					NSLog(@"Exhausted all seed for job %@", jobId);
					// we cannot now seal the job since we have to wait for all jobs to be finished
					// we therefore have to track which seeds were submitted and recognize
					// submission of results
					// we flag that we can now seal the job if all remaining seeds are done
					SetJobSeedsExhausted(jobId);
					break;
				}
				NSLog(@"Seed %@...", seedForJob);
				[self registerSeed:seedForJob forJob:jobId];
				NSLog(@"Dispatching unit for job %@ ...", jobId);
				[[freeClients objectAtIndex:k++] dispatchUnitFromSeed:seedForJob forJobId:jobId];
				NSLog(@"... dispatching done");
			}
		}
	}
	NS_HANDLER
		NSLog(@"Exception raised during job dispatch with reason: %@", [localException reason]);
	NS_ENDHANDLER
	NSLog(@"... did schedule");
}

- (void)terminateProxyForJobId:(NSString *)jobId
{
}

- (void)terminateJobProcessesForJobId:(NSString *)jobId
{
	[[clients allValues] makeObjectsPerformSelector:@selector(suspendJobWithId:)
		withObject:jobId];
	[self terminateProxyForJobId:jobId];
}

- (void)suspendJobStateToDiskJobId:(NSString *)jobId
{
	NSString	*jobPath = [[self serverTemporaryPath]
		stringByAppendingPathComponents:@"SuspendedJobs", jobId, nil];
	NSMutableDictionary	*jobSpec = [self jobSpecificationForJobId:jobId];

	[[NSFileManager defaultManager] createAllDirectoriesComposingPath:jobPath attributes:nil];

	/* rewire keys */
	[jobSpec setObject:[self dispatchedSeedsForJobId:jobId] forKey:SEEDSToBeRESCHEDULED_KEY];
	[self clearDispatchedSeedsSetForJobId:jobId];

	[jobSpec writeToFile:
		[jobPath stringByAppendingPathComponent:@"JobState.plist"] atomically:YES];
}

- (void)readJobStateFromDiskForJobId:(NSString *)jobId
{
	NSString	*jobPath = [[self serverTemporaryPath]
		stringByAppendingPathComponents:@"SuspendedJobs", jobId, nil];

	[self setJobSpecification:[NSDictionary dictionaryWithContentsOfFile:
		[jobPath stringByAppendingPathComponent:@"JobState.plist"]] forId:jobId];
}

- (void)reestablishObjectsForJobId:(NSString *)jobId
{
	NSMutableDictionary	*jobSpecification = [self jobSpecificationForJobId:jobId];

	id <ZillionJobProtocol>	job = [self loadJob:
			[jobSpecification objectForKey:PACKED_BUNDLE_KEY]
		withResources:
			[jobSpecification objectForKey:JOB_RESOURCES_KEY]];

	[jobSpecification setObject:job forKey:JOB_INSTANCE_KEY];
}

- (void)removeDataStructuresForJobId:(NSString *)jobId
{
	/* clear up specifications */
	[_jobs removeObjectForKey:jobId];
}


/**
 *	Job supspension is a three step process:
 *	1. Stop the job units currently running
 *		- inform the clients to do so
 *		- the clients in turn have to stop the corresponding threads
 *		- stop proxy processes for the job
 *	2. Store the job state on disk
 *		- seed id from enumerator
 *		- pending seeds
 *		- outcome dict
 *		- job specification
 *	3. Clean up data structures
 *		- job specifications
 *		- job outcomes
 */

- (void)suspendJobWithId:(NSString *)jobId
{
	/* Step 1 */
	[self terminateJobProcessesForJobId:jobId];

	/* Step 2 */
	[self suspendJobStateToDiskJobId:jobId];

	/* Step 3 */
	[self removeDataStructuresForJobId:jobId];

	RESCHEDULE();
}

- (void)resumeJobWithId:(NSString *)jobId
{
	[self readJobStateFromDiskForJobId:jobId];
	[self reestablishObjectsForJobId:jobId];
}

- (void)terminateJobWithId:(NSString *)jobId
{
	[self terminateJobProcessesForJobId:jobId];
	[self removeDataStructuresForJobId:jobId];

	RESCHEDULE();
}

- (void)setProtocolForProxy:(Protocol *)aProtocol
{
	NSLog(@"ZillionServer is asked to setProtocolForProxy: "
		@"though she doesn't feel inclined to");
}

@end
