lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-26 15:46:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

followup on HER-2031 ExtractorHTMLForms.java - improve regexes looking for form and input attributes, mainly to avoid maxing cpu for long periods of time for certain input

Browse Source
This commit is contained in:
Noah Levitt
2013-05-13 22:37:02 -07:00
parent 5b4d8c7f0d
commit bea5c6f267
1 changed files with 8 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/forms/ExtractorHTMLForms.java
+8 -10
View File
@@ -117,17 +117,15 @@ public class ExtractorHTMLForms extends Extractor {
protected boolean shouldProcess(CrawlURI uri) {
return uri.containsDataKey(ExtractorHTML.A_FORM_OFFSETS);
}
public void extract(CrawlURI curi) {
try {
ReplayCharSequence cs = curi.getRecorder().getContentReplayCharSequence();
// Extract all links from the charsequence
analyze(curi, cs);
// Set flag to indicate that link extraction is completed.
analyze(curi, cs);
} catch (IOException e) {
curi.getNonFatalFailures().add(e);
logger.log(Level.WARNING,"Failed get of replay char sequence in " +
Thread.currentThread().getName(), e);
Thread.currentThread().getName(), e);
}
}
@@ -145,15 +143,15 @@ public class ExtractorHTMLForms extends Extractor {
for (Object offset : curi.getDataList(ExtractorHTML.A_FORM_OFFSETS)) {
int offsetInt = (Integer) offset;
CharSequence relevantSequence = cs.subSequence(offsetInt, cs.length());
String method = findAttributeValueGroup("(?i)[^>]*\\smethod=([^>\\s]+)[^>]*?>",1,relevantSequence);
String action = findAttributeValueGroup("(?i)[^>]*\\saction=([^>\\s]+)[^>]*?>",1,relevantSequence);
String method = findAttributeValueGroup("(?i)^[^>]*\\smethod\\s*=\\s*([^>\\s]+)[^>]*>",1,relevantSequence);
String action = findAttributeValueGroup("(?i)^[^>]*\\saction\\s*=\\s*([^>\\s]+)[^>]*>",1,relevantSequence);
HTMLForm form = new HTMLForm();
form.setMethod(method);
form.setAction(action);
for(CharSequence input : findGroups("(?i)(<input\\s[^>]*>)|(</?form>)",1,relevantSequence)) {
String type = findAttributeValueGroup("(?i)[^>]*\\stype=([^>\\s]+)[^>]*?>",1,input);
String name = findAttributeValueGroup("(?i)[^>]*\\sname=([^>\\s]+)[^>]*?>",1,input);
String value = findAttributeValueGroup("(?i)[^>]*\\svalue=([^>\\s]+)[^>]*?>",1,input);
String type = findAttributeValueGroup("(?i)^[^>]*\\stype\\s*=\\s*([^>\\s]+)[^>]*>",1,input);
String name = findAttributeValueGroup("(?i)^[^>]*\\sname\\s*=\\s*([^>\\s]+)[^>]*>",1,input);
String value = findAttributeValueGroup("(?i)^[^>]*\\svalue\\s*=\\s*([^>\\s]+)[^>]*>",1,input);
form.addField(type,name,value);
}
if (form.seemsLoginForm() || getExtractAllForms()) {