| CODENOTIFIER | HelpYou are not signed inSign in |
Project: PUNKsearch
Revision: 538
Author: ysoldak
Date: 24 Aug 2008 18:30:42
Changes:Dumping HostCrawler statuses to punksearch-crawl.status was moved from cli to core.
Minor refactorings.
| ... | ...@@ -10,16 +10,9 @@ | |
| 10 | 10 | ***************************************************************************/ |
| 11 | 11 | package org.punksearch.cli; |
| 12 | 12 | |
| 13 | import java.io.File; | |
| 14 | 13 | import java.io.FileNotFoundException; |
| 15 | import java.io.IOException; | |
| 16 | import java.util.List; | |
| 17 | import java.util.Timer; | |
| 18 | import java.util.TimerTask; | |
| 19 | 14 | |
| 20 | import org.apache.commons.io.FileUtils; | |
| 21 | 15 | import org.punksearch.common.PunksearchProperties; |
| 22 | import org.punksearch.crawler.HostCrawler; | |
| 23 | 16 | import org.punksearch.crawler.NetworkCrawler; |
| 24 | 17 | |
| 25 | 18 | /** |
| ... | ...@@ -29,10 +22,6 @@ | |
| 29 | 22 | */ |
| 30 | 23 | public class CrawlerMain { |
| 31 | 24 | |
| 32 | public static String DUMP_STATUS_PERIOD = "org.punksearch.cli.dump.status.period"; | |
| 33 | ||
| 34 | private static long dumpPeriodSec = Long.getLong(DUMP_STATUS_PERIOD, 10L); | |
| 35 | ||
| 36 | 25 | public static void main(String[] args) throws InterruptedException { |
| 37 | 26 | try { |
| 38 | 27 | PunksearchProperties.loadDefault(); |
| ... | ...@@ -51,47 +40,7 @@ | |
| 51 | 40 | public void start() throws InterruptedException { |
| 52 | 41 | Thread crawlerThread = new Thread(NetworkCrawler.getInstance()); |
| 53 | 42 | crawlerThread.start(); |
| 54 | ||
| 55 | TimerTask dumpStatus = new StatusDumpTask(); | |
| 56 | ||
| 57 | Timer timer = new Timer(); | |
| 58 | timer.scheduleAtFixedRate(dumpStatus, dumpPeriodSec * 1000, dumpPeriodSec * 1000); | |
| 59 | ||
| 60 | 43 | crawlerThread.join(); |
| 61 | timer.cancel(); | |
| 62 | 44 | } |
| 63 | 45 | |
| 64 | private class StatusDumpTask extends TimerTask { | |
| 65 | ||
| 66 | public static final String STATUS_FILENAME = "punksearch-crawl.status"; | |
| 67 | ||
| 68 | public void run() { | |
| 69 | List<HostCrawler> threads = NetworkCrawler.getInstance().getThreads(); | |
| 70 | String dump = ""; | |
| 71 | for (HostCrawler thread : threads) { | |
| 72 | boolean stop = thread.isStopRequested(); | |
| 73 | String status = "unknown"; | |
| 74 | if (stop) { | |
| 75 | if (thread.getIp() != null) { | |
| 76 | status = "stopping"; | |
| 77 | } else { | |
| 78 | status = "stopped manually"; | |
| 79 | } | |
| 80 | } else { | |
| 81 | if (thread.getIp() != null) { | |
| 82 | status = "crawling " + thread.getIp(); | |
| 83 | } else { | |
| 84 | status = "finished successfully"; | |
| 85 | } | |
| 86 | } | |
| 87 | dump += thread.getName() + " : " + status + " : " + thread.getCrawledHosts().size() + "\n"; | |
| 88 | } | |
| 89 | try { | |
| 90 | String path = System.getProperty("java.io.tmpdir") + File.separator + STATUS_FILENAME; | |
| 91 | FileUtils.writeStringToFile(new File(path), dump); | |
| 92 | } catch (IOException e) { | |
| 93 | System.err.println("Can't write crawler status"); | |
| 94 | } | |
| 95 | } | |
| 96 | } | |
| 97 | 46 | } |