mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-22 13:47:02 +00:00
Merge pull request #742 from internetarchive/ato/BrowserProcessor-fixes
BrowserProcessor fixes
This commit is contained in:
4 files changed
+94
-13
No files matched your search
@@ -31,8 +31,10 @@ jobs:
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
- name: Show Firefox version
|
||||
run: firefox --version
|
||||
- name: Build with Maven
|
||||
run: mvn -B package --file pom.xml
|
||||
run: mvn -B package --file pom.xml -DrunBrowserTests=true
|
||||
env:
|
||||
# Apparently Azure kills idle http connections after 4 minutes which can cause the build to fail
|
||||
# with a timeout error. Limit the connection ttl to 2 minutes to workaround this as per
|
||||
|
||||
@@ -23,6 +23,8 @@ import org.apache.commons.lang3.StringUtils;
|
||||
import org.archive.crawler.event.CrawlURIDispositionEvent;
|
||||
import org.archive.crawler.framework.CrawlController;
|
||||
import org.archive.crawler.framework.Frontier;
|
||||
import org.archive.io.RecorderLengthExceededException;
|
||||
import org.archive.io.RecorderTimeoutException;
|
||||
import org.archive.io.RecordingInputStream;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.Processor;
|
||||
@@ -61,6 +63,7 @@ import java.time.Duration;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@@ -69,6 +72,8 @@ import static java.lang.System.Logger.Level.WARNING;
|
||||
import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.FAILED;
|
||||
import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.SUCCEEDED;
|
||||
import static org.archive.modules.CoreAttributeConstants.A_HTTP_RESPONSE_HEADERS;
|
||||
import static org.archive.modules.fetcher.FetchErrors.LENGTH_TRUNC;
|
||||
import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC;
|
||||
|
||||
/**
|
||||
* Opens a web page in a local web browser via WebDriver BiDi and runs {@link Behavior}s to interact with the page.
|
||||
@@ -171,7 +176,9 @@ public class BrowserProcessor extends Processor {
|
||||
String pageId = UUID.randomUUID().toString();
|
||||
var tab = webdriver.browsingContext().create(BrowsingContext.CreateType.tab).context();
|
||||
try {
|
||||
BrowserPage page = new BrowserPage(curi, new IdleBarrier(), webdriver, tab, fetcher.getProxy());
|
||||
String userAgent = curi.getUserAgent();
|
||||
if (userAgent == null) userAgent = fetcher.getUserAgentProvider().getUserAgent();
|
||||
BrowserPage page = new BrowserPage(curi, new IdleBarrier(), webdriver, tab, fetcher.getProxy(), userAgent);
|
||||
pages.put(pageId, page);
|
||||
pageIdsByContext.put(tab, pageId);
|
||||
webdriver.network().addIntercept(List.of(Network.InterceptPhase.beforeRequestSent), List.of(tab),
|
||||
@@ -285,7 +292,17 @@ public class BrowserProcessor extends Processor {
|
||||
if (event.request().url().startsWith("data:") || !event.isBlocked()) return;
|
||||
List<Network.Header> requestHeaders = event.request().headers();
|
||||
String pageId = pageIdsByContext.get(event.context());
|
||||
if (pageId != null) requestHeaders.add(new Network.Header(PAGE_ID_HEADER, pageId));
|
||||
if (pageId != null) {
|
||||
requestHeaders.add(new Network.Header(PAGE_ID_HEADER, pageId));
|
||||
BrowserPage page = pages.get(pageId);
|
||||
if (page != null) {
|
||||
String userAgent = page.userAgent();
|
||||
if (userAgent != null) {
|
||||
requestHeaders.removeIf(header -> header.name().equalsIgnoreCase("User-Agent"));
|
||||
requestHeaders.add(new Network.Header("User-Agent", userAgent));
|
||||
}
|
||||
}
|
||||
}
|
||||
webdriver.network().continueRequestAsync(event.request().request(), requestHeaders);
|
||||
}
|
||||
|
||||
@@ -371,6 +388,7 @@ public class BrowserProcessor extends Processor {
|
||||
private final BrowserPage page;
|
||||
private String method;
|
||||
private boolean recordingFailed;
|
||||
private boolean truncated;
|
||||
|
||||
public SubresourceRecorder(BrowserPage page, String url) throws IOException {
|
||||
this.page = page;
|
||||
@@ -381,7 +399,7 @@ public class BrowserProcessor extends Processor {
|
||||
crawlController.getRecorderOutBufferBytes(),
|
||||
crawlController.getRecorderInBufferBytes());
|
||||
this.curi.setRecorder(recorder);
|
||||
this.curi.setFetchBeginTime(System.currentTimeMillis());
|
||||
fetcher.beginRecording(curi, recorder);
|
||||
this.curi.getAnnotations().add("subresource");
|
||||
this.requestRecorder = Channels.newChannel(recorder.outputWrap(null));
|
||||
//noinspection resource
|
||||
@@ -429,8 +447,15 @@ public class BrowserProcessor extends Processor {
|
||||
|
||||
@Override
|
||||
public void onContent(org.eclipse.jetty.client.Response response, ByteBuffer content) {
|
||||
if (truncated) return; // already hit a limit; stop recording but treat as success
|
||||
try {
|
||||
responseRecorder.write(content.duplicate());
|
||||
} catch (RecorderLengthExceededException e) {
|
||||
curi.getAnnotations().add(LENGTH_TRUNC);
|
||||
truncated = true;
|
||||
} catch (RecorderTimeoutException e) {
|
||||
curi.getAnnotations().add(TIMER_TRUNC);
|
||||
truncated = true;
|
||||
} catch (Exception e) {
|
||||
logger.log(ERROR, "Error recording response body of " + curi, e);
|
||||
recordingFailed = true;
|
||||
@@ -445,6 +470,11 @@ public class BrowserProcessor extends Processor {
|
||||
|
||||
if (recordingFailed) {
|
||||
curi.setFetchStatus(FetchStatusCodes.S_RUNTIME_EXCEPTION);
|
||||
} else if (result.isFailed() && !truncated) {
|
||||
Throwable failure = result.getFailure();
|
||||
if (failure != null) curi.getNonFatalFailures().add(failure);
|
||||
curi.setFetchStatus(failure instanceof TimeoutException
|
||||
? FetchStatusCodes.S_TIMEOUT : FetchStatusCodes.S_CONNECT_FAILED);
|
||||
} else {
|
||||
subresourcesRecorded.incrementAndGet();
|
||||
curi.getOverlayNames(); // for sideeffect of creating the overlayNames list
|
||||
@@ -492,11 +522,14 @@ public class BrowserProcessor extends Processor {
|
||||
/**
|
||||
* A CrawlURI that's currently loaded as a page in a browser.
|
||||
*/
|
||||
protected record BrowserPage(CrawlURI curi,
|
||||
IdleBarrier networkActivity,
|
||||
WebDriverBiDi webdriver,
|
||||
BrowsingContext.Context context,
|
||||
ProxyConfiguration.Proxy proxy) implements Page {
|
||||
protected record BrowserPage(
|
||||
CrawlURI curi,
|
||||
IdleBarrier networkActivity,
|
||||
WebDriverBiDi webdriver,
|
||||
BrowsingContext.Context context,
|
||||
ProxyConfiguration.Proxy proxy,
|
||||
String userAgent
|
||||
) implements Page {
|
||||
|
||||
/**
|
||||
* Evaluates JavaScript and returns the result as simple Java objects (numbers, strings, maps, lists).
|
||||
|
||||
@@ -26,6 +26,7 @@ import java.util.*;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
import static java.lang.System.Logger.Level.DEBUG;
|
||||
import static org.archive.modules.fetcher.FetchErrors.LENGTH_TRUNC;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
@EnabledIfSystemProperty(named = "runBrowserTests", matches = "true")
|
||||
@@ -50,9 +51,13 @@ class BrowserProcessorTest {
|
||||
assertEquals(200, crawlURI.getFetchStatus());
|
||||
browserProcessor.innerProcess(crawlURI);
|
||||
|
||||
String expectedUserAgent = fetcher.getUserAgentProvider().getUserAgent();
|
||||
|
||||
var outLinks = new ArrayList<>(crawlURI.getOutLinks());
|
||||
assertEquals("/link", outLinks.get(0).getUURI().getPath());
|
||||
assertTrue(crawlURI.getAnnotations().contains("browser"));
|
||||
assertEquals(expectedUserAgent, crawlURI.getHttpResponseHeader("Reflected-User-Agent"));
|
||||
assertNotNull(crawlURI.getContentDigest());
|
||||
|
||||
logger.log(DEBUG, "Subrequests: {0}", subrequests);
|
||||
var subrequestByPath = new HashMap<String,CrawlURI>();
|
||||
@@ -70,6 +75,8 @@ class BrowserProcessorTest {
|
||||
assertTrue(postRequest.startsWith("POST /post-json HTTP/1.1\r\n"), postRequest);
|
||||
assertTrue(postRequest.toLowerCase(Locale.ROOT).contains("content-type: application/json\r\n"), postRequest);
|
||||
assertTrue(postRequest.endsWith("\r\n\r\n" + JSON_POST_BODY), postRequest);
|
||||
assertEquals(expectedUserAgent, postJson.getHttpResponseHeader("Reflected-User-Agent"));
|
||||
assertNotNull(postJson.getContentDigest());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -105,6 +112,30 @@ class BrowserProcessorTest {
|
||||
assertFalse(crawlURI.getAnnotations().contains("browser"), "navigation should have aborted");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTruncation() throws IOException, InterruptedException {
|
||||
fetcher.setMaxLengthBytes(10_000);
|
||||
try {
|
||||
CrawlURI crawlURI = newCrawlURI(baseUrl + "truncation");
|
||||
fetcher.process(crawlURI);
|
||||
assertEquals(200, crawlURI.getFetchStatus());
|
||||
browserProcessor.innerProcess(crawlURI);
|
||||
|
||||
var subrequestByPath = new HashMap<String,CrawlURI>();
|
||||
for (CrawlURI curi : subrequests) {
|
||||
subrequestByPath.put(curi.getUURI().getPath(), curi);
|
||||
}
|
||||
CrawlURI large = subrequestByPath.get("/large.bin");
|
||||
assertNotNull(large, "oversized subresource should still have been recorded");
|
||||
assertTrue(large.getAnnotations().contains(LENGTH_TRUNC),
|
||||
"oversized subresource should be annotated as length-truncated: " + large.getAnnotations());
|
||||
assertTrue(large.isSuccess(),
|
||||
"truncated subresource should still be a successful fetch, was status " + large.getFetchStatus());
|
||||
} finally {
|
||||
fetcher.setMaxLengthBytes(0L); // restore default (no limit)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHttpProxy() throws Exception {
|
||||
InetAddress localhost = Inet4Address.getLoopbackAddress();
|
||||
@@ -174,6 +205,8 @@ class BrowserProcessorTest {
|
||||
httpServer = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), -1);
|
||||
httpServer.createContext("/", exchange -> {
|
||||
logger.log(DEBUG, "Server received request: {0} {1}", exchange.getRequestMethod(), exchange.getRequestURI());
|
||||
String userAgent = exchange.getRequestHeaders().getFirst("User-Agent");
|
||||
if (userAgent != null) exchange.getResponseHeaders().add("Reflected-User-Agent", userAgent);
|
||||
String contentType = "text/html";
|
||||
int status = 200;
|
||||
String body = "";
|
||||
@@ -218,6 +251,11 @@ class BrowserProcessorTest {
|
||||
body = "sample-download-file";
|
||||
exchange.getResponseHeaders().add("Content-Disposition", "attachment; filename=heritrix-test.bin");
|
||||
}
|
||||
case "/truncation" -> body = "<img src=large.bin>";
|
||||
case "/large.bin" -> {
|
||||
body = "x".repeat(100_000);
|
||||
contentType = "application/octet-stream";
|
||||
}
|
||||
case "/gzip" -> {
|
||||
exchange.getResponseHeaders().add("Content-Encoding", "gzip");
|
||||
exchange.sendResponseHeaders(200, 0);
|
||||
|
||||
@@ -207,10 +207,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean
|
||||
var listener = new InputStreamResponseListener();
|
||||
|
||||
var recorder = curi.getRecorder();
|
||||
if (digestAlgorithm != null) recorder.getRecordedInput().setDigest(digestAlgorithm);
|
||||
recorder.getRecordedInput().setLimits(getMaxLengthBytes(),
|
||||
1000L * (long) getTimeoutSeconds(), getMaxFetchKBSec());
|
||||
curi.setFetchBeginTime(System.currentTimeMillis());
|
||||
beginRecording(curi, recorder);
|
||||
|
||||
try {
|
||||
UURI uuri = curi.getUURI();
|
||||
@@ -560,6 +557,17 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares the Recorder before a fetch begins: enables content digesting and applies the
|
||||
* length, time and rate limits. Also records the fetch begin time on the CrawlURI.
|
||||
*/
|
||||
public void beginRecording(CrawlURI curi, Recorder recorder) {
|
||||
if (digestAlgorithm != null) recorder.getRecordedInput().setDigest(digestAlgorithm);
|
||||
recorder.getRecordedInput().setLimits(getMaxLengthBytes(),
|
||||
1000L * (long) getTimeoutSeconds(), getMaxFetchKBSec());
|
||||
curi.setFetchBeginTime(System.currentTimeMillis());
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates the CrawlURI with details from the Recorder after it is closed.
|
||||
*/
|
||||
|
||||
Reference in new issue
Block a user