From 76d3847f359a2face58b24dd38bb2445c12af615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20K=C3=B6rner?= Date: Wed, 17 Dec 2025 18:26:52 +0100 Subject: [PATCH] Update documentation --- docs/api.rst | 217 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) diff --git a/docs/api.rst b/docs/api.rst index 17e08e06..4dd57367 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -61,6 +61,44 @@ Get Engine Status + **JSON Example:** + + .. code:: bash + + curl -v -k -u admin:admin --anyauth --location -H "Accept: application/json" https://localhost:8443/engine + + Response: + + .. code:: json + + { + "availableActions": [ + "rescan", + "add", + "create" + ], + "heapReport": { + "usedBytes": 69529904, + "totalBytes": 589824000, + "maxBytes": 2885681152 + }, + "jobsDirUrl": "https://localhost:8443/engine/jobsdir/", + "heritrixVersion": "3.3.0-SNAPSHOT-2017-07-12T04:17:56Z", + "jobsDir": "/heritrix/jobs", + "jobs": [{ + "isProfile": false, + "launchCount": 0, + "statusDescription": "Unbuilt", + "hasApplicationContext": false, + "shortName": "myjob", + "isLaunchInfoPartial": false, + "url": "https://localhost:8443/engine/job/myjob", + "key": "myjob", + "primaryConfig": "/heritrix/jobs/myjob/crawler-beans.cxml", + "primaryConfigUrl": "https://localhost:8443/engine/jobdir/crawler-beans.cxml" + }] + } + Create New Job ~~~~~~~~~~~~~~ @@ -86,6 +124,13 @@ Create New Job curl -v -d "createpath=myjob&action=create" -k -u admin:admin --anyauth --location \ -H "Accept: application/xml" https://localhost:8443/engine + **JSON Example:** + + .. code:: bash + + curl -v -d "createpath=myjob&action=create" -k -u admin:admin --anyauth --location \ + -H "Accept: application/json" https://localhost:8443/engine + Add Job Directory ~~~~~~~~~~~~~~~~~ @@ -110,6 +155,12 @@ Add Job Directory curl -v -d "action=add&addpath=/Users/hstern/job" -k -u admin:admin --anyauth --location -H "Accept: application/xml" https://localhost:8443/engine + **JSON Example:** + + .. code:: bash + + curl -v -d "action=add&addpath=/Users/hstern/job" -k -u admin:admin --anyauth --location -H "Accept: application/json" https://localhost:8443/engine + Get Job Status ~~~~~~~~~~~~~~ @@ -222,6 +273,112 @@ Get Job Status + **JSON Example:** + + .. code:: bash + + curl -v -k -u admin:admin --anyauth --location -H "Accept: application/json" https://localhost:8443/engine/job/myjob + + Response: + + .. code:: json + + { + "uriTotalsReport": { + "downloadedUriCount": 3920, + "queuedUriCount": 2, + "futureUriCount": 0, + "totalUriCount": 3920 + }, + "reports": [ + { + "shortName": "CrawlSummary", + "className": "CrawlSummaryReport" + }, + // ... + ], + "rateReport": { + "currentDocsPerSecond": 0.6354171124312226, + "averageKiBPerSec": 344, + "currentKiBPerSec": 0, + "averageDocsPerSecond": 0 + }, + "lastLaunch": "2020-04-01T02:07:42.531Z", + "frontierReport": { + "lastReachedState": "FINISH", + "totalQueues": 1, + "queueReadiedCount": 0, + "inProcessQueues": 0, + "readyQueues": 0, + "activeQueues": 0, + "inactiveQueues": 0, + "ineligibleQueues": 0, + "retiredQueues": 0, + "exhaustedQueues": 1, + "snoozedQueues": 0 + }, + "sizeTotalsReport": { + "notModifiedCount": 0, + "total": 2177235508, + "notModified": 0, + "dupByHashCount": 0, + "novelCount": 3920, + "sizeOnDisk": 0, + "totalCount": 3920, + "dupByHash": 0, + "novel": 2177235508, + "warcNovelContentBytes": 2177235508, + "warcNovelUrls": 3920 + }, + "checkpointFiles": [], + "crawlLogTail": [ + // ... + ], + "primaryConfig": "/heritrix/jobs/myjob/crawler-beans.cxml", + "primaryConfigUrl": "https://localhost:8443/engine/job/myjob/jobdir/crawler-beans.cxml", + "crawlLogFileUrl": "jobdir/20200401020744/logs/crawl.log", + "heapReport": { + "usedBytes": 66893400, + "totalBytes": 589824000, + "maxBytes": 2885681152 + }, + "isRunning": false, + "isLaunchable": false, + "isLaunchInfoPartial": false, + "alertLogFilePath": "/heritrix/jobs/myjob/20200401020744/logs/alerts.log", + "alertLogFileUrl": "jobdir/20200401020744/logs/alerts.log", + "availableActions": ["teardown"], + "launchCount": 1, + "isProfile": false, + "loadReport": { + "totalThreads": 0, + "busyThreads": 0, + "averageQueueDepth": 0, + "deepestQueueDepth": 0, + "congestionRatio": null + }, + "crawlExitStatus": "FINISHED", + "jobLogTail": [ + "2020-04-01T03:50:44.708Z INFO FINISHED 20200401020744", + "2020-04-01T03:50:42.670Z INFO EMPTY 20200401020744", + "2020-04-01T03:50:42.669Z INFO STOPPING 20200401020744" + ], + "url": "https://localhost:8443/engine/job/myjob/job/myjob", + "elapsedReport": { + "elapsedPretty": "1h42m49s176ms", + "elapsedMilliseconds": 6169176 + }, + "statusDescription": "Finished: FINISHED", + "configFiles": [ + // ... + ], + "hasApplicationContext": true, + "alertCount": 549, + "crawlLogFilePath": "/heritrix/jobs/myjob/20200401020744/logs/crawl.log", + "shortName": "myjob", + "crawlControllerState": "FINISHED" + } + Build Job Configuration ~~~~~~~~~~~~~~~~~~~~~~~ @@ -245,6 +402,12 @@ Build Job Configuration curl -v -d "action=build" -k -u admin:admin --anyauth --location -H "Accept: application/xml" https://localhost:8443/engine/job/myjob + **JSON Example:** + + .. code:: bash + + curl -v -d "action=build" -k -u admin:admin --anyauth --location -H "Accept: application/json" https://localhost:8443/engine/job/myjob + Launch Job ~~~~~~~~~~ @@ -270,6 +433,12 @@ Launch Job curl -v -d "action=launch" -k -u admin:admin --anyauth --location -H "Accept: application/xml" https://localhost:8443/engine/job/myjob + **JSON Example:** + + .. code:: bash + + curl -v -d "action=launch" -k -u admin:admin --anyauth --location -H "Accept: application/json" https://localhost:8443/engine/job/myjob + Rescan Job Directory ~~~~~~~~~~~~~~~~~~~~ @@ -293,6 +462,12 @@ Rescan Job Directory curl -v -d "action=rescan" -k -u admin:admin --anyauth --location -H "Accept: application/xml" https://localhost:8443/engine + **JSON Example:** + + .. code:: bash + + curl -v -d "action=rescan" -k -u admin:admin --anyauth --location -H "Accept: application/json" https://localhost:8443/engine + Pause Job ~~~~~~~~~ @@ -314,6 +489,12 @@ Pause Job curl -v -d "action=pause" -k -u admin:admin --anyauth --location -H "Accept: application/xml" https://localhost:8443/engine/job/myjob + **JSON Example:** + + .. code:: bash + + curl -v -d "action=pause" -k -u admin:admin --anyauth --location -H "Accept: application/json" https://localhost:8443/engine/job/myjob + Unpause Job ~~~~~~~~~~~ @@ -336,6 +517,12 @@ Unpause Job curl -v -d "action=unpause" -k -u admin:admin --anyauth --location -H "Accept: application/xml" https://localhost:8443/engine/job/myjob + **JSON Example:** + + .. code:: bash + + curl -v -d "action=unpause" -k -u admin:admin --anyauth --location -H "Accept: application/json" https://localhost:8443/engine/job/myjob + Terminate Job ~~~~~~~~~~~~~ @@ -357,6 +544,12 @@ Terminate Job curl -v -d "action=terminate" -k -u admin:admin --anyauth --location -H "Accept: application/xml" https://localhost:8443/engine/job/myjob + **JSON Example:** + + .. code:: bash + + curl -v -d "action=terminate" -k -u admin:admin --anyauth --location -H "Accept: application/json" https://localhost:8443/engine/job/myjob + Teardown Job ~~~~~~~~~~~~ @@ -379,6 +572,12 @@ Teardown Job curl -v -d "action=teardown" -k -u admin:admin --anyauth --location -H "Accept: application/xml" https://localhost:8443/engine/job/myjob + **JSON Example:** + + .. code:: bash + + curl -v -d "action=teardown" -k -u admin:admin --anyauth --location -H "Accept: application/json" https://localhost:8443/engine/job/myjob + Copy Job ~~~~~~~~ @@ -406,6 +605,12 @@ Copy Job curl -v -d "copyTo=mycopy&asProfile=on" -k -u admin:admin --anyauth --location -H "Accept: application/xml" https://localhost:8443/engine/job/myjob + **JSON Example:** + + .. code:: bash + + curl -v -d "copyTo=mycopy&asProfile=on" -k -u admin:admin --anyauth --location -H "Accept: application/json" https://localhost:8443/engine/job/myjob + Checkpoint Job ~~~~~~~~~~~~~~ @@ -429,6 +634,12 @@ Checkpoint Job curl -v -d "action=checkpoint" -k -u admin:admin --anyauth --location -H "Accept: application/xml" https://localhost:8443/engine/job/myjob + **JSON Example:** + + .. code:: bash + + curl -v -d "action=checkpoint" -k -u admin:admin --anyauth --location -H "Accept: application/json" https://localhost:8443/engine/job/myjob + Execute Script in Job ~~~~~~~~~~~~~~~~~~~~~ @@ -454,6 +665,12 @@ Execute Script in Job curl -v -d "engine=beanshell&script=System.out.println%28%22test%22%29%3B" -k -u admin:admin --anyauth --location -H "Accept: application/xml" https://localhost:8443/engine/job/myjob/script + **JSON Example:** + + .. code:: bash + + curl -v -d "engine=beanshell&script=System.out.println%28%22test%22%29%3B" -k -u admin:admin --anyauth --location -H "Accept: application/json" https://localhost:8443/engine/job/myjob/script + Submitting a CXML Job Configuration File ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~