From df7d3b506c221b66beeb4e1a892a5a7b4c9e9a14 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Fri, 5 Sep 2014 12:59:17 -0700 Subject: [PATCH 1/2] do not require "+" (plus sign) before @OPERATOR_CONTACT_URL@ in user-agent -- many bots including googlebot have the +, but some don't, so having it in the default template seems sufficient --- modules/src/main/java/org/archive/modules/CrawlMetadata.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/src/main/java/org/archive/modules/CrawlMetadata.java b/modules/src/main/java/org/archive/modules/CrawlMetadata.java index 6390ff27..f0e7f115 100644 --- a/modules/src/main/java/org/archive/modules/CrawlMetadata.java +++ b/modules/src/main/java/org/archive/modules/CrawlMetadata.java @@ -201,7 +201,7 @@ implements UserAgentProvider, protected static Validator VALIDATOR = new BeanFieldsPatternValidator( CrawlMetadata.class, "userAgentTemplate", - "^.*\\+@OPERATOR_CONTACT_URL@.*$", + "^.*\\@OPERATOR_CONTACT_URL@.*$", "You must supply a userAgentTemplate value that includes " + "the string \"@OPERATOR_CONTACT_URL@\" where your crawl" + "contact URL will appear.", From 98a6c08a8695c2cc7b7c03f1f419866dd88f60fa Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Fri, 5 Sep 2014 14:38:14 -0700 Subject: [PATCH 2/2] remove extraneous backslashes (thanks Kris) --- modules/src/main/java/org/archive/modules/CrawlMetadata.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/src/main/java/org/archive/modules/CrawlMetadata.java b/modules/src/main/java/org/archive/modules/CrawlMetadata.java index f0e7f115..9d810ce4 100644 --- a/modules/src/main/java/org/archive/modules/CrawlMetadata.java +++ b/modules/src/main/java/org/archive/modules/CrawlMetadata.java @@ -201,7 +201,7 @@ implements UserAgentProvider, protected static Validator VALIDATOR = new BeanFieldsPatternValidator( CrawlMetadata.class, "userAgentTemplate", - "^.*\\@OPERATOR_CONTACT_URL@.*$", + "^.*@OPERATOR_CONTACT_URL@.*$", "You must supply a userAgentTemplate value that includes " + "the string \"@OPERATOR_CONTACT_URL@\" where your crawl" + "contact URL will appear.",