diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java index 42379adf..8bc74da5 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java @@ -19,17 +19,39 @@ package org.archive.modules.extractor; +import org.archive.crawler.event.CrawlURIDispositionEvent; +import org.archive.crawler.framework.CrawlController; +import org.archive.crawler.framework.Frontier; import org.archive.modules.CrawlURI; import org.archive.net.chrome.ChromeClient; import org.archive.net.chrome.ChromeProcess; +import org.archive.net.chrome.ChromeRequest; import org.archive.net.chrome.ChromeWindow; +import org.archive.spring.KeyedProperties; +import org.archive.util.Recorder; import org.json.JSONArray; +import org.springframework.context.ApplicationEventPublisher; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.util.Arrays; import java.util.concurrent.ExecutionException; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +import static java.nio.charset.StandardCharsets.US_ASCII; +import static java.util.Collections.enumeration; +import static java.util.logging.Level.INFO; +import static java.util.logging.Level.WARNING; +import static java.util.regex.Pattern.CASE_INSENSITIVE; +import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.FAILED; +import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.SUCCEEDED; +import static org.archive.modules.CrawlURI.FetchType.*; /** * Extracts links using a web browser via the Chrome Devtools Protocol. @@ -37,6 +59,7 @@ import java.util.concurrent.TimeoutException; * To use, first define this as a top-level bean: *
* <bean id="extractorChrome" class="org.archive.modules.extractor.ExtractorChrome"> + * <!-- <property name="captureRequests" value="true" /> --> * <!-- <property name="devtoolsUrl" value="ws://127.0.0.1:1234/devtools/browser/2bc831e8-6c02-4c9b-affd-14c93b8579d7" /> --> * <!-- <property name="executable" value="chromium-browser" /> --> * <!-- <property name="loadTimeoutSeconds" value="30" /> --> @@ -52,6 +75,10 @@ import java.util.concurrent.TimeoutException; *--headless --remote-debugging-port=1234). */ public class ExtractorChrome extends ContentExtractor { + private static final Logger logger = Logger.getLogger(ExtractorChrome.class.getName()); + + private static final Pattern TRANSFER_ENCODING_RE = Pattern.compile("\r\nTransfer-Encoding:[^\n\r]+", CASE_INSENSITIVE); + /** * The maximum number of browser windows that are allowed to be opened simultaneously. Feel free to increase this * if you have lots of RAM available. @@ -83,10 +110,23 @@ public class ExtractorChrome extends ContentExtractor { */ private int loadTimeoutSeconds = 30; + /** + * Capture requests made by the browser. + */ + private boolean captureRequests = true; + private Semaphore openWindowsSemaphore = null; private ChromeProcess process = null; private ChromeClient client = null; + private final CrawlController controller; + private final ApplicationEventPublisher eventPublisher; + + public ExtractorChrome(CrawlController controller, ApplicationEventPublisher eventPublisher) { + this.controller = controller; + this.eventPublisher = eventPublisher; + } + @Override protected boolean shouldExtract(CrawlURI uri) { return uri.getContentType().startsWith("text/html"); @@ -107,25 +147,105 @@ public class ExtractorChrome extends ContentExtractor { return false; } - private void visit(CrawlURI uri) throws InterruptedException { + private void visit(CrawlURI curi) throws InterruptedException { try (ChromeWindow window = client.createWindow(windowWidth, windowHeight)) { + if (captureRequests) { + window.captureRequests(request -> handleCapturedRequest(curi, request)); + } + try { - window.navigateAsync(uri.getURI()).get(loadTimeoutSeconds, TimeUnit.SECONDS); + window.navigateAsync(curi.getURI()).get(loadTimeoutSeconds, TimeUnit.SECONDS); } catch (ExecutionException e) { throw new RuntimeException(e.getCause()); } catch (TimeoutException e) { - throw new RuntimeException("Timed out navigating to " + uri.getURI()); + throw new RuntimeException("Timed out navigating to " + curi.getURI()); } JSONArray links = window.eval("Array.from(document.querySelectorAll('a[href], area[href]'))" + ".map(link => link.protocol + '//' + link.host + link.pathname + link.search + link.hash)") .getJSONArray("value"); for (int i = 0; i < links.length(); i++) { - addOutlink(uri, links.getString(i), LinkContext.NAVLINK_MISC, Hop.NAVLINK); + addOutlink(curi, links.getString(i), LinkContext.NAVLINK_MISC, Hop.NAVLINK); } } } + private void handleCapturedRequest(CrawlURI via, ChromeRequest request) { + Recorder recorder = new Recorder(controller.getScratchDir().getFile(), + controller.getRecorderOutBufferBytes(), + controller.getRecorderInBufferBytes()); + try { + String digestAlgorithm = "sha1"; + recorder.getRecordedInput().setDigest(digestAlgorithm); + recorder.getRecordedOutput().write(request.getRequestHeader().getBytes(US_ASCII)); + recorder.getRecordedOutput().write(request.getRequestBody()); + + // strip the Transfer-Encoding header since we only have access to the decoded body + String responseHeader = TRANSFER_ENCODING_RE.matcher(request.getResponseHeader()).replaceAll(""); + + recorder.inputWrap(new SequenceInputStream(enumeration(Arrays.asList( + new ByteArrayInputStream(responseHeader.getBytes(US_ASCII)), + new InputStream() { + public int read() { + recorder.markContentBegin(); + return -1; + } + }, + new ByteArrayInputStream(request.getResponseBody()))))); + recorder.getRecordedInput().readFully(); + recorder.closeRecorders(); + + CrawlURI curi = via.createCrawlURI(request.getUrl(), LinkContext.EMBED_MISC, Hop.EMBED); + curi.getAnnotations().add("browser"); + curi.setContentDigest(digestAlgorithm, recorder.getRecordedInput().getDigestValue()); + curi.setContentSize(recorder.getRecordedInput().getSize()); + curi.setContentType(request.getResponseContentType()); + curi.setFetchBeginTime(request.getBeginTime()); + curi.setFetchCompletedTime(System.currentTimeMillis()); + curi.setFetchStatus(request.getStatus()); + curi.setRecorder(recorder); + curi.setServerIP(request.getRemoteIPAddress()); + curi.setThreadNumber(via.getThreadNumber()); + + switch (request.getMethod()) { + case "GET": + curi.setFetchType(HTTP_GET); + break; + case "POST": + curi.setFetchType(HTTP_POST); + break; + default: + curi.setFetchType(UNKNOWN); + break; + } + + // send it to the disposition chain to invoke the warc writer etc + Frontier frontier = controller.getFrontier(); // allowed to be null to simplify unit tests + curi.getOverlayNames(); // for side-effect of creating the overlayNames list + KeyedProperties.loadOverridesFrom(curi); + try { + frontier.beginDisposition(curi); + controller.getDispositionChain().process(curi,null); + } finally { + KeyedProperties.clearOverridesFrom(curi); + } + + curi.aboutToLog(); + controller.getLoggerModule().getUriProcessing().log(INFO, curi.getUURI().toString(), curi); + + if (curi.isSuccess()) { + eventPublisher.publishEvent(new CrawlURIDispositionEvent(this, curi, SUCCEEDED)); + } else { + eventPublisher.publishEvent(new CrawlURIDispositionEvent(this, curi, FAILED)); + } + frontier.endDisposition(); + } catch (Exception e) { + logger.log(WARNING, "Exception handling subrequest " + request.getUrl(), e); + } finally { + recorder.cleanup(); + } + } + @Override public void start() { if (isRunning) return; diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java b/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java index 7de68b3a..cc81598b 100644 --- a/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java +++ b/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; +import java.util.logging.Level; import java.util.logging.Logger; import static java.util.logging.Level.WARNING; @@ -43,7 +44,6 @@ public class ChromeClient implements Closeable { private final DevtoolsSocket devtoolsSocket; private final AtomicLong nextMessageId = new AtomicLong(0); private final Map> responseFutures = new ConcurrentHashMap<>(); - private final ExecutorService threadPool = Executors.newCachedThreadPool(); final ConcurrentHashMap > sessionEventHandlers = new ConcurrentHashMap<>(); public ChromeClient(String devtoolsUrl) { @@ -110,16 +110,7 @@ public class ChromeClient implements Closeable { String sessionId = message.getString("sessionId"); Consumer handler = sessionEventHandlers.get(sessionId); if (handler != null) { - // Run event handlers on a different thread so we don't block the websocket receiving thread. - // That would cause a deadlock if an event handler itself made an RPC call as the response could - // never be processed. - threadPool.submit(() -> { - try { - handler.accept(message); - } catch (Throwable t) { - logger.log(WARNING, "Exception handling browser event " + message, t); - } - }); + handler.accept(message); } else { logger.log(WARNING, "Received event for unknown session {0}", sessionId); } @@ -164,13 +155,14 @@ public class ChromeClient implements Closeable { } @Override - public void onClose(int i, String s, boolean b) { - + public void onClose(int code, String reason, boolean remote) { + if (!remote) return; + logger.log(WARNING, "Websocket closed by browser: " + reason); } @Override public void onError(Exception e) { - + logger.log(Level.SEVERE, "Websocket error", e); } } } diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeRequest.java b/contrib/src/main/java/org/archive/net/chrome/ChromeRequest.java new file mode 100644 index 00000000..68ce072b --- /dev/null +++ b/contrib/src/main/java/org/archive/net/chrome/ChromeRequest.java @@ -0,0 +1,154 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.net.chrome; + +import org.json.JSONObject; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +public class ChromeRequest { + private final ChromeWindow window; + private final String id; + private JSONObject requestJson; + private JSONObject rawRequestHeaders; + private JSONObject responseJson; + private JSONObject rawResponseHeaders; + private String responseHeadersText; + private final long beginTime = System.currentTimeMillis(); + + public ChromeRequest(ChromeWindow window, String id) { + this.window = window; + this.id = id; + } + + void setRawRequestHeaders(JSONObject rawRequestHeaders) { + this.rawRequestHeaders = rawRequestHeaders; + } + + void setResponseJson(JSONObject responseJson) { + this.responseJson = responseJson; + } + + void setRawResponseHeaders(JSONObject headers) { + this.rawResponseHeaders = headers; + } + + void setResponseHeadersText(String responseHeadersText) { + this.responseHeadersText = responseHeadersText; + } + + public String getUrl() { + return requestJson.getString("url"); + } + + public byte[] getResponseBody() { + JSONObject reply = window.call("Network.getResponseBody", "requestId", id); + byte[] body; + if (reply.getBoolean("base64Encoded")) { + body = Base64.getDecoder().decode(reply.getString("body")); + } else { + body = reply.getString("body").getBytes(StandardCharsets.UTF_8); + } + return body; + } + + public String getRequestHeader() { + if (responseJson != null && responseJson.has("requestHeadersText")) { + return responseJson.getString("requestHeadersText"); + } + StringBuilder builder = new StringBuilder(); + builder.append(requestJson.getString("method")); + builder.append(' '); + builder.append(getUrl()); + builder.append(" HTTP/1.1\r\n"); + formatHeaders(builder, requestJson.getJSONObject("headers"), rawRequestHeaders); + return builder.toString(); + } + + public byte[] getRequestBody() { + if (requestJson.has("postData")) { + return requestJson.getString("postData").getBytes(StandardCharsets.UTF_8); + } else { + return new byte[0]; + } + } + + public String getResponseHeader() { + if (responseHeadersText != null) { + return responseHeadersText; + } else if (responseJson.has("headersText")) { + return responseJson.getString("headersText"); + } + StringBuilder builder = new StringBuilder(); + if (responseJson.getString("protocol").equals("http/1.0")) { + builder.append("HTTP/1.0"); + } else { + builder.append("HTTP/1.1"); + } + builder.append(getStatus()); + builder.append(" "); + builder.append(responseJson.getString("statusText")); + builder.append("\r\n"); + formatHeaders(builder, responseJson.getJSONObject("headers"), rawResponseHeaders); + return builder.toString(); + } + + private void formatHeaders(StringBuilder builder, JSONObject headers, JSONObject rawHeaders) { + if (rawHeaders != null) { + headers = rawHeaders; + } + for (Object key : headers.keySet()) { + builder.append(key); + builder.append(": "); + builder.append(headers.getString((String) key)); + builder.append("\r\n"); + } + builder.append("\r\n"); + } + + public String getMethod() { + return requestJson.getString("method"); + } + + public int getStatus() { + return responseJson.getInt("status"); + } + + public String getResponseContentType() { + return responseJson.getString("mimeType"); + } + + public long getBeginTime() { + return beginTime; + } + + public String getRemoteIPAddress() { + if (responseJson.has("remoteIPAddress")) { + return responseJson.getString("remoteIPAddress"); + } else { + return null; + } + } + + void setRequestJson(JSONObject requestJson) { + this.requestJson = requestJson; + } +} diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java b/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java index 59e8f2e7..a65fa8cc 100644 --- a/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java +++ b/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java @@ -22,10 +22,13 @@ package org.archive.net.chrome; import org.json.JSONObject; import java.io.Closeable; -import java.util.concurrent.CompletableFuture; +import java.util.Map; +import java.util.concurrent.*; +import java.util.function.Consumer; import java.util.logging.Logger; import static java.util.logging.Level.FINE; +import static java.util.logging.Level.WARNING; /** * A browser window or tab. @@ -38,12 +41,17 @@ public class ChromeWindow implements Closeable { private final String sessionId; private boolean closed; private CompletableFuture loadEventFuture; + private final Map requestMap = new ConcurrentHashMap<>(); + private Consumer requestConsumer; + private final ExecutorService eventExecutor; public ChromeWindow(ChromeClient client, String targetId) { this.client = client; this.targetId = targetId; this.sessionId = client.call("Target.attachToTarget", "targetId", targetId, "flatten", true).getString("sessionId"); + eventExecutor = Executors.newSingleThreadExecutor(runnable -> + new Thread(runnable, "ChromeWindow (sessionId=" + sessionId +")")); client.sessionEventHandlers.put(sessionId, this::handleEvent); call("Page.enable"); // for loadEventFired call("Page.setLifecycleEventsEnabled", "enabled", true); // for networkidle @@ -78,9 +86,47 @@ public class ChromeWindow implements Closeable { } private void handleEvent(JSONObject message) { + // Run event handlers on a different thread so we don't block the websocket receiving thread. + // That would cause a deadlock if an event handler itself made an RPC call as the response could + // never be processed. + // We use a single thread per window though as the order events are processed is important. + eventExecutor.submit(() -> { + try { + handleEventOnEventThread(message); + } catch (Throwable t) { + logger.log(WARNING, "Exception handling browser event " + message, t); + } + }); + } + + private void handleEventOnEventThread(JSONObject message) { + JSONObject params = message.getJSONObject("params"); switch (message.getString("method")) { + case "Fetch.requestPaused": + handlePausedRequest(params); + break; + case "Network.requestWillBeSent": + handleRequestWillBeSent(params); + break; + case "Network.requestWillBeSentExtraInfo": + handleRequestWillBeSentExtraInfo(params); + break; + case "Network.responseReceived": + handleResponseReceived(params); + break; + case "Network.responseReceivedExtraInfo": + handleResponseReceivedExtraInfo(params); + break; + case "Network.loadingFinished": + handleLoadingFinished(params); + break; case "Page.loadEventFired": if (loadEventFuture != null) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } loadEventFuture.complete(null); } break; @@ -90,11 +136,77 @@ public class ChromeWindow implements Closeable { } } + private void handleRequestWillBeSent(JSONObject params) { + String requestId = params.getString("requestId"); + ChromeRequest request = requestMap.computeIfAbsent(requestId, id -> new ChromeRequest(this, id)); + request.setRequestJson(params.getJSONObject("request")); + } + + private void handleRequestWillBeSentExtraInfo(JSONObject params) { + // it seems this event can arrive both before and after requestWillBeSent so we need to cope with that + String requestId = params.getString("requestId"); + ChromeRequest request = requestMap.computeIfAbsent(requestId, id -> new ChromeRequest(this, id)); + request.setRawRequestHeaders(params.getJSONObject("headers")); + } + + private void handleResponseReceived(JSONObject params) { + ChromeRequest request = requestMap.get(params.getString("requestId")); + if (request == null) { + logger.log(WARNING, "Got responseReceived event without corresponding requestWillBeSent"); + return; + } + request.setResponseJson(params.getJSONObject("response")); + } + + private void handleResponseReceivedExtraInfo(JSONObject params) { + ChromeRequest request = requestMap.get(params.getString("requestId")); + if (request == null) { + logger.log(WARNING, "Got responseReceivedExtraInfo event without corresponding requestWillBeSent"); + return; + } + request.setRawResponseHeaders(params.getJSONObject("headers")); + request.setResponseHeadersText(params.getString("headersText")); + } + + private void handleLoadingFinished(JSONObject params) { + ChromeRequest request = requestMap.get(params.getString("requestId")); + if (request == null) { + logger.log(WARNING, "Got loadingFinished event without corresponding requestWillBeSent"); + return; + } + if (requestConsumer != null) { + requestConsumer.accept(request); + } + } + @Override public void close() { if (closed) return; closed = true; + eventExecutor.shutdown(); + try { + eventExecutor.awaitTermination(1, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } client.call("Target.closeTarget", "targetId", targetId); client.sessionEventHandlers.remove(sessionId); } + + public void captureRequests(Consumer requestConsumer) { + this.requestConsumer = requestConsumer; + call("Network.enable"); + } + + private void handlePausedRequest(JSONObject params) { + if (params.has("responseStatusCode")) { + String stream = call("Fetch.takeResponseBodyAsStream", "requestId", + params.getString("requestId")).getString("stream"); + System.out.println(call("IO.read", "handle", stream)); + System.out.println("stream " + stream); + call("IO.close", "handle", stream); + } else { + call("Fetch.continueRequest", "requestId", params.getString("requestId")); + } + } } \ No newline at end of file diff --git a/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java b/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java index ba77fa21..17a9db14 100644 --- a/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java +++ b/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java @@ -19,34 +19,144 @@ package org.archive.modules.extractor; -import org.apache.commons.httpclient.URIException; +import org.archive.crawler.framework.CrawlController; +import org.archive.crawler.framework.Frontier; +import org.archive.crawler.reporting.CrawlerLoggerModule; import org.archive.modules.CrawlURI; +import org.archive.modules.DispositionChain; +import org.archive.modules.Processor; import org.archive.net.UURIFactory; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.handler.AbstractHandler; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import static java.util.stream.Collectors.toList; -import static org.junit.Assert.assertEquals; +import static org.archive.modules.CrawlURI.FetchType.HTTP_GET; +import static org.archive.modules.CrawlURI.FetchType.HTTP_POST; +import static org.easymock.EasyMock.*; +import static org.junit.Assert.*; import static org.junit.Assume.assumeNoException; public class ExtractorChromeTest { + private static Server server; + + @BeforeClass + public static void startServer() throws Exception { + server = new Server(InetSocketAddress.createUnresolved("127.0.0.1", 7778)); + server.setHandler(new AbstractHandler() { + @Override + public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { + switch (target) { + case "/": + response.setContentType("text/html"); + response.getWriter().write("link" + + " " + + ""); + baseRequest.setHandled(true); + break; + case "/blue.png": + response.setContentType("image/png"); + response.getWriter().write("bogus png"); + baseRequest.setHandled(true); + break; + case "/post": + response.setContentType("plain/text"); + response.getWriter().write("method=" + request.getMethod()); + baseRequest.setHandled(true); + break; + default: + System.err.println("Unhandled target: " + target); + break; + } + + } + }); + server.start(); + } + + @AfterClass + public static void stopServer() throws Exception { + server.stop(); + } + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + @Test - public void test() throws URIException { - ExtractorChrome extractor = new ExtractorChrome(); + public void test() throws IOException { + List
processedURIs = Collections.synchronizedList(new ArrayList<>()); + CrawlController controller = new CrawlController(); + DispositionChain dispositionChain = new DispositionChain(); + dispositionChain.setProcessors(Arrays.asList(new Processor() { + @Override + protected boolean shouldProcess(CrawlURI uri) { + return true; + } + + @Override + protected void innerProcess(CrawlURI uri) { + processedURIs.add(uri); + } + })); + controller.setDispositionChain(dispositionChain); + controller.setLoggerModule(new CrawlerLoggerModule() { + @Override + public Logger getUriProcessing() { + Logger logger = Logger.getAnonymousLogger(); + logger.setLevel(Level.WARNING); + return logger; + } + }); + Frontier frontier = createMock(Frontier.class); + frontier.considerIncluded(anyObject()); + expectLastCall().anyTimes(); + frontier.beginDisposition(anyObject()); + expectLastCall().anyTimes(); + frontier.endDisposition(); + expectLastCall().anyTimes(); + frontier.finished(anyObject()); + expectLastCall().anyTimes(); + replay(frontier); + controller.setFrontier(frontier); + + ExtractorChrome extractor = new ExtractorChrome(controller, event -> { /* ignored */ }); try { extractor.start(); } catch (RuntimeException e) { assumeNoException("Unable to start Chrome", e); } try { - CrawlURI curi = new CrawlURI(UURIFactory.getInstance("data:text/html,link")); + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://127.0.0.1:7778/")); extractor.innerExtract(curi); List outLinks = curi.getOutLinks().stream().map(CrawlURI::toString).sorted().collect(toList()); assertEquals(Collections.singletonList("http://example.org/page2.html"), outLinks); } finally { extractor.stop(); } + + assertEquals(3, processedURIs.size()); + for (CrawlURI curi: processedURIs) { + assertEquals(200, curi.getFetchStatus()); + assertEquals(curi.getUURI().getPath().equals("/post") ? HTTP_POST : HTTP_GET, curi.getFetchType()); + assertNotNull(curi.getContentDigest()); + assertTrue(curi.getContentSize() > 0); + } } } \ No newline at end of file