Author: olegk
Date: Sat Nov 8 02:06:05 2008
New Revision: 712370
URL: http://svn.apache.org/viewvc?rev=712370&view=rev
Log:
Moved Norobots to a separate module
Added:
incubator/droids/trunk/module-norobots/pom.xml
incubator/droids/trunk/module-norobots/src/main/java/org/apache/droids/norobots/
- copied from r712100,
incubator/droids/trunk/module-core/src/main/java/org/apache/droids/norobots/
Removed:
incubator/droids/trunk/module-core/src/main/java/org/apache/droids/norobots/
Modified:
incubator/droids/trunk/module-core/pom.xml
incubator/droids/trunk/module-norobots/ (props changed)
incubator/droids/trunk/module-norobots/src/main/java/org/apache/droids/norobots/NoRobotClient.java
incubator/droids/trunk/pom.xml
Modified: incubator/droids/trunk/module-core/pom.xml
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/module-core/pom.xml?rev=712370&r1=712369&r2=712370&view=diff
==============================================================================
--- incubator/droids/trunk/module-core/pom.xml (original)
+++ incubator/droids/trunk/module-core/pom.xml Sat Nov 8 02:06:05 2008
@@ -49,6 +49,11 @@
<dependencies>
<dependency>
+ <groupId>org.apache.droids</groupId>
+ <artifactId>droids-norobots</artifactId>
+ <version>${pom.version}</version>
+ </dependency>
+ <dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</dependency>
Propchange: incubator/droids/trunk/module-norobots/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Sat Nov 8 02:06:05 2008
@@ -0,0 +1,9 @@
+classes
+bin
+lib
+target
+log.txt
+.classpath
+.project
+.settings
+
Added: incubator/droids/trunk/module-norobots/pom.xml
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/module-norobots/pom.xml?rev=712370&view=auto
==============================================================================
--- incubator/droids/trunk/module-norobots/pom.xml (added)
+++ incubator/droids/trunk/module-norobots/pom.xml Sat Nov 8 02:06:05 2008
@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<!--
+ ====================================================================
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ ====================================================================
+
+ This software consists of voluntary contributions made by many
+ individuals on behalf of the Apache Software Foundation. For more
+ information on the Apache Software Foundation, please see
+ <http://www.apache.org />.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/maven-v4_0_0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>droids</artifactId>
+ <groupId>org.apache.droids</groupId>
+ <version>0.1-incubating-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <artifactId>droids-norobots</artifactId>
+ <name>Droids Norobots</name>
+ <inceptionYear>2007</inceptionYear>
+ <description>
+ Droids robots.txt parser
+ </description>
+ <packaging>jar</packaging>
+
+ <scm>
+
<connection>scm:svn:http://svn.apache.org/repos/asf/incubator/droids/trunk/module-norobots</connection>
+
<developerConnection>scm:svn:https://svn.apache.org/repos/asf/incubator/droids/trunk/module-norobots</developerConnection>
+
<url>http://svn.apache.org/repos/asf/labs/incubator/trunk/module-norobots</url>
+ </scm>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>${maven.compile.source}</source>
+ <target>${maven.compile.target}</target>
+ <optimize>${maven.compile.optimize}</optimize>
+ <showDeprecations>${maven.compile.deprecation}</showDeprecations>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
Modified:
incubator/droids/trunk/module-norobots/src/main/java/org/apache/droids/norobots/NoRobotClient.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/module-norobots/src/main/java/org/apache/droids/norobots/NoRobotClient.java?rev=712370&r1=712100&r2=712370&view=diff
==============================================================================
---
incubator/droids/trunk/module-norobots/src/main/java/org/apache/droids/norobots/NoRobotClient.java
(original)
+++
incubator/droids/trunk/module-norobots/src/main/java/org/apache/droids/norobots/NoRobotClient.java
Sat Nov 8 02:06:05 2008
@@ -36,8 +36,6 @@
import java.net.HttpURLConnection;
import java.net.URLConnection;
-import org.apache.droids.net.UrlHelper;
-
/**
* A Client which may be used to decide which urls on a website
* may be looked at, according to the norobots specification
@@ -69,11 +67,10 @@
* @param baseUrl of the site
*/
public void parse(URL baseUrl) throws NoRobotException {
-
URL txtUrl = null;
try {
// fetch baseUrl+"robots.txt"
- txtUrl = UrlHelper.findRobotsUrl(baseUrl);
+ txtUrl = new URL(baseUrl, "/robots.txt");
} catch(MalformedURLException murle) {
throw new NoRobotException("Bad URL: "+baseUrl+", robots.txt. ", murle);
}
Modified: incubator/droids/trunk/pom.xml
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/pom.xml?rev=712370&r1=712369&r2=712370&view=diff
==============================================================================
--- incubator/droids/trunk/pom.xml (original)
+++ incubator/droids/trunk/pom.xml Sat Nov 8 02:06:05 2008
@@ -131,6 +131,7 @@
</dependencyManagement>
<modules>
+ <module>module-norobots</module>
<module>module-core</module>
<module>depot/dynamic</module>
</modules>
