Added: nutch/trunk/src/java/org/apache/nutch/webui/service/NutchService.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/service/NutchService.java?rev=1705744&view=auto ============================================================================== --- nutch/trunk/src/java/org/apache/nutch/webui/service/NutchService.java (added) +++ nutch/trunk/src/java/org/apache/nutch/webui/service/NutchService.java Mon Sep 28 18:58:33 2015 @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.webui.service; + +import java.util.Map; + +import org.apache.nutch.webui.client.model.ConnectionStatus; +import org.apache.nutch.webui.client.model.NutchStatus; + +public interface NutchService { + public ConnectionStatus getConnectionStatus(Long instanceId); + + public Map<String, String> getNutchConfig(Long instanceId); + + public NutchStatus getNutchStatus(Long instanceId); + +}
Added: nutch/trunk/src/java/org/apache/nutch/webui/service/SeedListService.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/service/SeedListService.java?rev=1705744&view=auto ============================================================================== --- nutch/trunk/src/java/org/apache/nutch/webui/service/SeedListService.java (added) +++ nutch/trunk/src/java/org/apache/nutch/webui/service/SeedListService.java Mon Sep 28 18:58:33 2015 @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.webui.service; + +import java.util.List; + +import org.apache.nutch.webui.model.SeedList; + +public interface SeedListService { + + public void save(SeedList seedList); + + public void delete(Long seedListId); + + public List<SeedList> findAll(); + + public SeedList getSeedList(Long seedListId); + +} Added: nutch/trunk/src/java/org/apache/nutch/webui/service/impl/CrawlServiceImpl.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/service/impl/CrawlServiceImpl.java?rev=1705744&view=auto ============================================================================== --- nutch/trunk/src/java/org/apache/nutch/webui/service/impl/CrawlServiceImpl.java (added) +++ nutch/trunk/src/java/org/apache/nutch/webui/service/impl/CrawlServiceImpl.java Mon Sep 28 18:58:33 2015 @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.webui.service.impl; + +import java.sql.SQLException; +import java.util.List; + +import javax.annotation.Resource; + +import org.apache.nutch.webui.client.NutchClient; +import org.apache.nutch.webui.client.NutchClientFactory; +import org.apache.nutch.webui.client.impl.CrawlingCycle; +import org.apache.nutch.webui.client.impl.RemoteCommandsBatchFactory; +import org.apache.nutch.webui.client.impl.CrawlingCycleListener; +import org.apache.nutch.webui.client.impl.RemoteCommand; +import org.apache.nutch.webui.client.impl.RemoteCommandExecutor; +import org.apache.nutch.webui.client.model.Crawl; +import org.apache.nutch.webui.client.model.Crawl.CrawlStatus; +import org.apache.nutch.webui.model.NutchInstance; +import org.apache.nutch.webui.service.CrawlService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; + +import com.j256.ormlite.dao.Dao; + +@Service +public class CrawlServiceImpl implements CrawlService, CrawlingCycleListener { + private Logger log = LoggerFactory.getLogger(CrawlServiceImpl.class); + + @Resource + private Dao<Crawl, Long> crawlDao; + + @Resource + private NutchClientFactory nutchClientFactory; + + @Resource + private RemoteCommandsBatchFactory commandFactory; + + @Override + @Async + public void startCrawl(Long crawlId, NutchInstance instance) { + Crawl crawl = null; + try { + crawl = crawlDao.queryForId(crawlId); + if(crawl.getCrawlId()==null) { + crawl.setCrawlId("crawl-" + crawlId.toString()); + } + NutchClient client = nutchClientFactory.getClient(instance); + String seedDirectory = client.createSeed(crawl.getSeedList()); + crawl.setSeedDirectory(seedDirectory); + + List<RemoteCommand> commands = commandFactory.createCommands(crawl); + RemoteCommandExecutor executor = new RemoteCommandExecutor(client); + + CrawlingCycle cycle = new CrawlingCycle(this, executor, crawl, commands); + cycle.executeCrawlCycle(); + + } catch (Exception e) { + crawl.setStatus(CrawlStatus.ERROR); + saveCrawl(crawl); + log.error("exception occured", e); + } + } + + @Override + public List<Crawl> getCrawls() { + try { + return crawlDao.queryForAll(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void saveCrawl(Crawl crawl) { + try { + crawlDao.createOrUpdate(crawl); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void deleteCrawl(Long crawlId) { + try { + crawlDao.deleteById(crawlId); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void crawlingStarted(Crawl crawl) { + crawl.setStatus(CrawlStatus.CRAWLING); + crawl.setProgress(0); + saveCrawl(crawl); + } + + @Override + public void onCrawlError(Crawl crawl, String msg) { + crawl.setStatus(CrawlStatus.ERROR); + saveCrawl(crawl); + } + + @Override + public void commandExecuted(Crawl crawl, RemoteCommand command, int progress) { + crawl.setProgress(progress); + saveCrawl(crawl); + } + + @Override + public void crawlingFinished(Crawl crawl) { + crawl.setStatus(CrawlStatus.FINISHED); + saveCrawl(crawl); + } +} Added: nutch/trunk/src/java/org/apache/nutch/webui/service/impl/NutchInstanceServiceImpl.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/service/impl/NutchInstanceServiceImpl.java?rev=1705744&view=auto ============================================================================== --- nutch/trunk/src/java/org/apache/nutch/webui/service/impl/NutchInstanceServiceImpl.java (added) +++ nutch/trunk/src/java/org/apache/nutch/webui/service/impl/NutchInstanceServiceImpl.java Mon Sep 28 18:58:33 2015 @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.webui.service.impl; + +import java.sql.SQLException; +import java.util.List; + +import javax.annotation.Resource; + +import org.apache.nutch.webui.client.NutchClientFactory; +import org.apache.nutch.webui.model.NutchInstance; +import org.apache.nutch.webui.service.NutchInstanceService; +import org.springframework.stereotype.Service; + +import com.j256.ormlite.dao.Dao; + +@Service +public class NutchInstanceServiceImpl implements NutchInstanceService { + + @Resource + private NutchClientFactory nutchClientFactory; + + @Resource + private Dao<NutchInstance, Long> instancesDao; + + @Override + public List<NutchInstance> getInstances() { + try { + return instancesDao.queryForAll(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + + } + + @Override + public NutchInstance getInstance(Long id) { + try { + return instancesDao.queryForId(id); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void saveInstance(NutchInstance instance) { + try { + instancesDao.createOrUpdate(instance); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void removeInstance(Long id) { + try { + instancesDao.deleteById(id); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } +} Added: nutch/trunk/src/java/org/apache/nutch/webui/service/impl/NutchServiceImpl.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/service/impl/NutchServiceImpl.java?rev=1705744&view=auto ============================================================================== --- nutch/trunk/src/java/org/apache/nutch/webui/service/impl/NutchServiceImpl.java (added) +++ nutch/trunk/src/java/org/apache/nutch/webui/service/impl/NutchServiceImpl.java Mon Sep 28 18:58:33 2015 @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.webui.service.impl; + +import java.net.ConnectException; +import java.util.Collections; +import java.util.Map; + +import javax.annotation.Resource; + +import org.apache.nutch.webui.client.NutchClientFactory; +import org.apache.nutch.webui.client.model.ConnectionStatus; +import org.apache.nutch.webui.client.model.NutchStatus; +import org.apache.nutch.webui.model.NutchInstance; +import org.apache.nutch.webui.service.NutchInstanceService; +import org.apache.nutch.webui.service.NutchService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import com.sun.jersey.api.client.ClientHandlerException; + +@Service +public class NutchServiceImpl implements NutchService { + private static final Logger logger = LoggerFactory + .getLogger(NutchServiceImpl.class); + + @Resource + private NutchClientFactory nutchClientFactory; + + @Resource + private NutchInstanceService instanceService; + + @Override + public ConnectionStatus getConnectionStatus(Long instanceId) { + NutchInstance instance = instanceService.getInstance(instanceId); + try { + NutchStatus nutchStatus = nutchClientFactory.getClient(instance) + .getNutchStatus(); + if (nutchStatus.getStartDate() != null) { + return ConnectionStatus.CONNECTED; + } + } catch (Exception e) { + if (e.getCause() instanceof ConnectException) { + return ConnectionStatus.DISCONNECTED; + } + + logger.error("Cannot connect to nutch server!", e); + } + return null; + } + + @Override + public Map<String, String> getNutchConfig(Long instanceId) { + NutchInstance instance = instanceService.getInstance(instanceId); + try { + return nutchClientFactory.getClient(instance).getNutchConfig("default"); + } catch (ClientHandlerException exception) { + return Collections.emptyMap(); + } + } + + @Override + public NutchStatus getNutchStatus(Long instanceId) { + NutchInstance instance = instanceService.getInstance(instanceId); + return nutchClientFactory.getClient(instance).getNutchStatus(); + } +} Added: nutch/trunk/src/java/org/apache/nutch/webui/service/impl/SeedListServiceImpl.java URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/webui/service/impl/SeedListServiceImpl.java?rev=1705744&view=auto ============================================================================== --- nutch/trunk/src/java/org/apache/nutch/webui/service/impl/SeedListServiceImpl.java (added) +++ nutch/trunk/src/java/org/apache/nutch/webui/service/impl/SeedListServiceImpl.java Mon Sep 28 18:58:33 2015 @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.webui.service.impl; + +import java.sql.SQLException; +import java.util.List; + +import javax.annotation.Resource; + +import org.apache.nutch.webui.model.SeedList; +import org.apache.nutch.webui.model.SeedUrl; +import org.apache.nutch.webui.service.SeedListService; +import org.springframework.stereotype.Service; + +import com.j256.ormlite.dao.Dao; + +@Service +public class SeedListServiceImpl implements SeedListService { + + @Resource + private Dao<SeedList, Long> seedListDao; + + @Resource + private Dao<SeedUrl, Long> seedUrlDao; + + @Override + public void save(SeedList seedList) { + try { + seedListDao.createOrUpdate(seedList); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void delete(Long seedListId) { + try { + seedListDao.deleteById(seedListId); + } catch (SQLException e) { + throw new RuntimeException(e); + } + + } + + @Override + public List<SeedList> findAll() { + try { + return seedListDao.queryForAll(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public SeedList getSeedList(Long seedListId) { + try { + return seedListDao.queryForId(seedListId); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + +}
