heritrix3

lee/heritrix3

0 Fork 0

mirror of https://github.com/internetarchive/heritrix3.git synced 2026-11-03 04:30:48 +00:00

Commit Graph

Select branches

Hide Pull Requests

ARI-5439

adam/add-crawl-log-logging-to-extractoryoutubedl

adam/add_configurable_extractor_js_script_attribute_rules

adam/ait-add-crawl-log-logging-to-extractoryoutubedl

adam/fix_amqp_consumer_leak

adam/fix_configurable_js_extractor_strict_overlay

adam/fix_crawl_end_time_checkpoint_resumption

adam/knowledgable_extractorjs_extends_configurable_extractorjs

adam/restore-has-prerequisite-behavior

adam/strip_invalid_xml_chars_from_rest_api_output

adds-bdbmodule-max-file-size-option

adds-subqueue-support-forced-queue-assignment

adds-trough-dedup-performance-rework

adds-trough-local-dedup-write-cache

ait-qa

ait-ydl-updates

collectd-socket-stats-writer

dependabot/github_actions/actions/setup-java-6.1.0

fix-invoker-command-fallback

fix-yt-dlp-typo-too

fixes-WARCWriterChainProcessor-checkpoint-stats

fixes-extractor-multiple-regex-matcher-recycle

fixes-leaky-file-handles

fixes-prerequisite-status

fixes-trough-client-synchronization

fixes-trough-crawl-log-feed-synchronization

less_youtubedl

master

master-ait-contrib

master-it

meta-content-name

noplaylist-ydl

paged-log-viewer-search

restlet_npe

revert-367-fixes-trough-crawl-log-feed-synchronization

ydl-no-cache

yt-dl-format

yt-dl-format-medium-ish

yt-dlp

yt-dlp-20230706

yt-dlp-4-h3

yt-dlp-for-h3

#1

#1/merge

#10

#10/merge

#100

#101

#102

#103

#104

#105

#106

#107

#108

#109

#11

#11/merge

#110

#111

#112

#113

#114

#115

#116

#117

#118

#119

#12

#12/merge

#120

#121

#122

#123

#124

#125

#126

#127

#127/merge

#128

#129

#13

#13/merge

#130

#131

#132

#133

#134

#134/merge

#135

#136

#137

#138

#139

#14

#14/merge

#140

#140/merge

#141

#142

#143

#144

#145

#146

#147

#148

#149

#15

#15/merge

#150

#151

#152

#153

#153/merge

#154

#159

#16

#16/merge

#161

#163

#164

#165

#166

#168

#169

#17

#17/merge

#171

#174

#174/merge

#178

#179

#18

#180

#183

#183/merge

#184

#184/merge

#185

#186

#187

#188

#188/merge

#189

#19

#19/merge

#190

#192

#194

#196

#197

#2

#2/merge

#20

#20/merge

#205

#207

#209

#21

#21/merge

#210

#213

#215

#216

#217

#218

#219

#22

#22/merge

#220

#222

#223

#224

#226

#227

#23

#230

#231

#235

#238

#24

#240

#241

#242

#243

#244

#246

#248

#249

#25

#251

#253

#255

#256

#257

#26

#260

#262

#269

#27

#270

#271

#272

#273

#276

#28

#280

#281

#283

#285

#286

#287

#288

#289

#29

#290

#292

#292/merge

#294

#295

#296

#298

#3

#3/merge

#30

#300

#301

#302

#303

#304

#306

#308

#31

#310

#312

#313

#314

#315

#317

#32

#320

#323

#325

#326

#327

#328

#329

#33

#330

#331

#333

#334

#335

#336

#34

#341

#342

#345

#346

#347

#348

#349

#35

#352

#354

#355

#356

#357

#359

#36

#362

#364

#365

#366

#367

#368

#369

#37

#370

#373

#374

#375

#377

#379

#38

#381

#382

#383

#384

#385

#386

#388

#388/merge

#389

#39

#390

#391

#393

#394

#395

#397

#398

#399

#4

#4/merge

#40

#400

#401

#402

#403

#404

#405

#406

#407

#408

#409

#41

#410

#411

#412

#414

#415

#416

#418

#42

#421

#423

#424

#43

#431

#433

#434

#435

#44

#441

#442

#444

#445

#448

#45

#450

#452

#454

#456

#457

#459

#46

#460

#461

#465

#468

#47

#470

#475

#476

#478

#48

#480

#483

#484

#486

#488

#489

#49

#490

#491

#492

#493

#5

#5/merge

#50

#503

#51

#52

#53

#534

#535

#536

#537

#538

#539

#54

#542

#543

#544

#549

#55

#55/merge

#550

#551

#552

#554

#555

#56

#565

#566

#567

#57

#572

#573

#574

#575

#576

#577

#578

#579

#58

#580

#581

#582

#583

#586

#589

#59

#590

#591

#592

#593

#594

#595

#597

#598

#599

#6

#6/merge

#60

#600

#601

#602

#603

#605

#606

#607

#608

#609

#61

#610

#611

#612

#614

#615

#616

#617

#618

#619

#62

#621

#622

#623

#624

#625

#626

#627

#628

#629

#63

#630

#631

#632

#638

#639

#64

#640

#642

#643

#644

#645

#646

#647

#648

#649

#65

#650

#651

#652

#653

#654

#655

#656

#657

#659

#66

#660

#661

#663

#664

#665

#667

#668

#669

#67

#670

#671

#672

#673

#674

#675

#676

#677

#678

#679

#68

#680

#681

#682

#683

#684

#685

#686

#69

#691

#692

#693

#694

#695

#696

#697

#7

#7/merge

#70

#700

#701

#703

#704

#705

#706

#707

#708

#709

#71

#710

#712

#713

#719

#72

#721

#722

#723

#725

#728

#73

#730

#731

#732

#733

#734

#735

#736

#737

#738

#739

#74

#740

#741

#742

#744

#745

#746

#75

#750

#751

#755

#756

#757

#758

#759

#76

#760

#761

#761/merge

#762

#763

#764

#765

#766

#767

#77

#770

#771

#772

#774

#775

#776

#777

#778

#779

#78

#780

#781

#782

#783

#784

#785

#786

#787

#788

#79

#790

#791

#792

#8

#8/merge

#80

#81

#82

#83

#84

#85

#85/merge

#86

#87

#88

#89

#9

#9/merge

#90

#91

#92

#93

#94

#95

#96

#97

#98

#98/merge

#99

3.0.0

3.1.1

3.10.0

3.10.1

3.10.2

3.11.0

3.12.0

3.13.0

3.14.0

3.14.1

3.15.0

3.16.0

3.17.0

3.17.1

3.18.0

3.2.0

3.4.0-20190205

3.4.0-20190207

3.4.0-20190418

3.4.0-20200304

3.4.0-20200518

3.4.0-20210527

3.4.0-20210617

3.4.0-20210618

3.4.0-20210621

3.4.0-20210803

3.4.0-20210923

3.4.0-20220727

3.4.0-20240909

3.5.0

3.6.0

3.7.0

3.8.0

3.9.0

87a15bddc8 Merge pull request #168 from internetarchive/gh-167 Noah Levitt 2017-06-07 18:15:19 -07:00
c001470ad7 Merge branch 'aitfive-1176' into ait-qa Noah Levitt 2017-06-07 18:13:26 -07:00
66fabf790e hacked up tarball with heritrix, contrib and dependencies Noah Levitt 2017-06-07 17:45:41 -07:00
eed0210a8f batch trough writes Noah Levitt 2017-06-07 13:28:27 -07:00
d1452c1d21 initial version of TroughCrawlLogFeed.java Noah Levitt 2017-06-07 11:09:36 -07:00
a4bb99e828 Merge pull request #179 from BitBaron/aitfive-1225 Noah Levitt 2017-06-02 12:01:17 -07:00
e5d49c491f Merge branch 'aitfive-1225' into ait-qa Neil Minton 2017-06-02 11:58:15 -07:00
10541184b5 Clean up after code review. Neil Minton 2017-05-18 11:20:01 -05:00
4c0e52af61 Add parsing for srcset attributes. Neil Minton 2017-05-01 17:08:15 -07:00
4a14211ee6 Merge branch 'master' into ait-qa Neil Minton 2017-05-01 17:13:43 -07:00
752ea028f6 fix floating point format Noah Levitt 2017-03-09 10:41:31 -08:00
d13bfb6a37 Remove unit tests which moved over from Heritrix3 to webarchive-commons Mohamed Elsayed 2017-03-09 14:19:11 +02:00
548ea5eb84 Merge pull request #178 from internetarchive/kafka-mem-hog Barbara Miller 2017-03-07 11:11:23 -08:00
bd6607739d KafkaCrawlLogFeed had been using lots of heap because each callback instance was keeping around a CrawlURI and in some cases there could be many of thousands of callbacks waiting around; instead, just keep some stats and log them periodically Noah Levitt 2017-02-13 17:29:13 -08:00
0ca88c6071 Merge pull request #171 from ukwa/amqp-fine-control Noah Levitt 2016-10-13 17:10:29 -07:00
7e965f3512 No need to do anything for the usual link types as the FrontierPreparer handles them. Andrew Jackson 2016-10-11 20:56:46 +01:00
a4a841c826 Removing comment from old version of this code. Andrew Jackson 2016-10-11 20:43:32 +01:00
b3dff8d9d6 Defaulting the message prefetch count to 1000 instead of infinity. Andrew Jackson 2016-10-11 20:42:52 +01:00
4d7a64d127 Reverted Kryo version change. Andrew Jackson 2016-10-11 20:41:20 +01:00
b3e1a83034 Merge branch 'amqp-fine-control' of github.com:ukwa/heritrix3 into amqp-fine-control Andrew Jackson 2016-10-03 16:52:48 +01:00
0e40869f83 Fix package name. Andrew Jackson 2016-05-23 12:11:15 +01:00
fdae4b6297 Update Kryo to latest version. Andrew Jackson 2016-05-23 11:00:59 +01:00
eaec682dd4 Bump incoming links to MEDIUM to compete with e.g. runs of redirects. Andrew Jackson 2016-03-13 20:22:27 +00:00
c6c396af8b Use sensible defaults for normal links. Andrew Jackson 2016-03-10 14:41:40 +00:00
553eae2ddd Add support for setting the hop and the prefetch count. Andrew Jackson 2016-02-12 11:20:51 +00:00
4d6969554c Merge branch 'bitbaron/master' into ait-qa Neil Minton 2016-08-19 19:49:38 -07:00
8c78297ded Add annotation for reject decisions to write crawl information. Neil Minton 2016-08-19 19:38:04 -07:00
2ee5db7f3b Formating fixes in POM file. Neil Minton 2016-08-19 19:36:09 -07:00
0b4671c291 fix for race-condition when first using the WARC writers https://github.com/internetarchive/heritrix3/issues/167 Noah Levitt 2016-08-15 13:36:33 -07:00
fbb0173459 Merge pull request #166 from internetarchive/amqpWaiterStarter Noah Levitt 2016-07-21 18:05:10 -05:00
b5fdd49ec8 check controller state, not frontier.isEmpty) Barbara Miller 2016-07-21 11:40:58 -07:00
cbc73a6e8d check controller state, not frontier.isEmpty) Barbara Miller 2016-07-21 11:40:58 -07:00
63dda8bd8b don't wait if h3 sends no url Barbara Miller 2016-07-12 23:40:10 -07:00
40d8ee77bf don't wait if h3 sends no url Barbara Miller 2016-07-12 23:40:10 -07:00
0060adec29 minor merge tweaks Barbara Miller 2016-07-12 22:45:03 -07:00
af2530edc4 Merge pull request #165 from galgeek/amqpWaiter Noah Levitt 2016-07-07 19:35:04 -05:00
4485b8c8c7 AMQP URL Waiter Barbara Miller 2016-06-23 19:21:57 -07:00
264b32ab56 autowired afterall Barbara Miller 2016-06-28 19:21:25 -07:00
1542f04e13 3rd time? Barbara Miller 2016-06-28 19:01:20 -07:00
c85e2072ad more better frontier checking Barbara Miller 2016-06-28 18:14:16 -07:00
c90bea7b7c better frontier checking Barbara Miller 2016-06-28 17:37:43 -07:00
c8ece72d3c tweak Barbara Miller 2016-06-26 18:58:48 -07:00
9071e669ad public AMQPUrlWaiter(); Barbara Miller 2016-06-26 18:26:47 -07:00
1f7ff71319 last cannot find symbols Barbara Miller 2016-06-24 17:38:00 -07:00
5400e31594 better Event params Barbara Miller 2016-06-24 17:14:37 -07:00
0d5853f72c missing imports Barbara Miller 2016-06-24 17:00:45 -07:00
6b6c689df1 move AMQPURLReceivedEvent.java under contrib Barbara Miller 2016-06-24 16:00:03 -07:00
1d83d47b07 AMQP URL Waiter Barbara Miller 2016-06-23 19:21:57 -07:00
c04162a0f7 Merge pull request #164 from nlevitt/fix-test-errors Noah Levitt 2016-06-08 14:31:58 -05:00
36ea1a9fb2 try to fix remaining serialization tests in oraclejdk8 by using ConcurrentSkipListMap instead of ConcurrentHashMap Noah Levitt 2016-06-07 15:33:09 -05:00
99d87e8f23 hopefully fix serialization tests in oraclejdk8 by using TreeSet instead of HashSet in KeyedProperties.java Noah Levitt 2016-06-07 14:53:11 -05:00
cf9498d2ed clear the history store at the beginning of testBasics(), because the other test might have run first Noah Levitt 2016-06-07 14:12:20 -05:00
b09b078075 yeesh... "cd .." to get back to the right place to see the failure reports Noah Levitt 2016-06-07 13:36:45 -05:00
bc8b0baa59 Merge remote-tracking branch 'origin/master' into fix-test-errors Noah Levitt 2016-06-07 13:09:51 -05:00
ac41da69c2 let me see what failed, travis Noah Levitt 2016-06-07 13:07:24 -05:00
ca3b65ed03 exclude dns from stats tests because the size of the dns records can vary depending on the environment Noah Levitt 2016-06-06 15:43:02 -05:00
f7d1ad7f1f Merge pull request #111 from nlevitt/bean-browse-cookiestore-exception Noah Levitt 2016-06-06 11:11:45 -07:00
bb10b4a80c Merge pull request #161 from kris-sigur/NoNonSuccessLinkExtraction Noah Levitt 2016-06-06 11:11:06 -07:00
aba94c7ac1 Fix package name. Andrew Jackson 2016-05-23 12:11:15 +01:00
d349c19298 Update Kryo to latest version. Andrew Jackson 2016-05-23 11:00:59 +01:00
c80f0cee2b Merge pull request #163 from nlevitt/kafka-0.9 jkafader 2016-05-09 12:01:53 -07:00
ecd27e9e65 Setup TravisCI nruest 2016-04-14 07:07:11 -04:00
70867efdac Set fetch status on curis when testing link extraction Kristinn Sigurðsson 2016-05-03 15:58:50 +00:00
08e3d055cf No link extraction on URI not successfully downloaded Kristinn Sigurðsson 2016-05-03 14:40:24 +00:00
c5848ae803 Fixes issue #158 : Noisy alerts about 401s without auth challenge (#159) Kristinn Sigurðsson 2016-05-02 17:15:32 +00:00
8437a2724c Merge pull request #3 from ruebot/fix-test-errors Andy Jackson 2016-04-14 17:54:37 +00:00
3ee34d886f Setup TravisCI nruest 2016-04-14 07:07:11 -04:00
a02a32cdfe Make Content-Location header url INFERRED not REFFER hop type since Content-Location is not for redirection (#151) Hunter 2016-04-13 15:19:21 -07:00
48f5e71e76 Add delay to avoid race condition during test. Andrew Jackson 2016-04-07 21:48:20 +01:00
07d610a1a7 Merge branch 'fix-test-errors' into amqp-fine-control Andrew Jackson 2016-04-02 20:58:43 +01:00
9145fedd40 Fix unexpected change in count of novel bytes in one selftest. Andrew Jackson 2016-04-02 20:57:05 +01:00
cdb79b8868 Fix serialisation tests by defaulting to TreeMaps. Andrew Jackson 2016-04-02 20:35:37 +01:00
621468107b Bump incoming links to MEDIUM to compete with e.g. runs of redirects. Andrew Jackson 2016-03-13 20:22:27 +00:00
576b1d234b Use sensible defaults for normal links. Andrew Jackson 2016-03-10 14:41:40 +00:00
1341a61a3c Merge branch 'master' into amqp-fine-control Andrew Jackson 2016-03-09 19:48:57 +00:00
124907341f Merge pull request #152 from ukwa/switch-to-java-7 Noah Levitt 2016-03-08 16:47:58 -08:00
8ff09dc769 Switch to Java 1.7 Andrew Jackson 2016-03-08 23:53:59 +00:00
e1476401ac Merged with upstream changes. Andrew Jackson 2016-03-08 23:50:07 +00:00
7f752d8d45 Merge pull request #150 from nlevitt/ari-4638 Adam Miller 2016-03-08 09:26:58 -08:00
0dc95ab7f7 Merge pull request #144 from nlevitt/ari-4528 Hunter 2016-02-29 14:24:43 -08:00
ae413b1d9e Merge pull request #149 from eleclerc/patch-1 Kristinn Sigurðsson 2016-02-19 09:49:34 +00:00
9bdafc3c33 remove stray line Noah Levitt 2016-02-18 18:19:44 -08:00
bc9a6cc640 new parameter extraInfo, "Arbitrary additional information to include in the json payload", on AMQPPublishProcessor; used KeyedProperties so parameters can be overridden with the sheet/overlay system Noah Levitt 2016-02-18 18:17:30 -08:00
f4b8468564 set custom http headers last, so they override headers set elsewhere (e.g. cookies) Noah Levitt 2016-02-18 18:16:03 -08:00
a64551296a new boolean forceFetch setting; REQUEST_HEADER_BLACKLIST to exclude certain http headers, since with https://github.com/internetarchive/umbra/pull/55 umbra sends over more stuff Noah Levitt 2016-02-18 18:15:22 -08:00
8a6a49f7fc ARI-4528 avoid this exception Noah Levitt 2016-02-17 17:45:21 -08:00
5f431de215 Update to ExtractorHTML.java for cond. comments Eric Leclerc 2016-02-17 14:49:50 -05:00
847400776b Merge pull request #148 from vonrosen/ARI-4738 Noah Levitt 2016-02-17 11:43:30 -08:00
3e219af148 Set source tag to uri as it exists before possible prepending of scheme. Hunter Stern 2016-02-17 08:10:07 -08:00
b9b2343bc0 Don't canonicalize source tag so that SourceSeedDecideRule will work. Trailing slash of canonicalized seed currently causes it not to work. Hunter Stern 2016-02-16 08:52:16 -08:00
7b3c9e51e3 Add support for setting the hop and the prefetch count. Andrew Jackson 2016-02-12 11:20:51 +00:00
bfbd61edd7 Merge pull request #145 from vonrosen/ARI-4713 Hunter 2016-02-11 08:27:52 -08:00
cbb5e3ab99 Simplify logic for urls with spaces, and make it better follow the pattern of UriUtils.isVeryLikelyUri(). Involves a subtle adjustment to the regex LIKELY_RELATIVE_URI_PATTERN to ensure the 2nd capturing group always gets the file extension. Also add a couple of tests of strings with spaces and file extensions that are not known good extensions. Noah Levitt 2016-02-09 15:02:11 -08:00
d80cf8244b Merge pull request #146 from vonrosen/ARI-4656-2 Noah Levitt 2016-02-09 10:48:07 -08:00
30ce2f09a0 Take care of possible NullPointerException Hunter Stern 2016-02-09 08:22:39 -08:00
44139e7e3b fixes for kafka 0.9 (?) Noah Levitt 2016-02-08 16:08:00 -08:00
f665ce844f upgrade to kafka 0.9 Noah Levitt 2016-02-08 12:49:47 -08:00
e7c46753ce Fix formatting Hunter Stern 2016-01-29 18:26:18 -08:00
e83ff04e86 More fixes for https://webarchive.jira.com/browse/ARI-4656 Hunter Stern 2016-01-29 17:25:06 -08:00
088189ee8b Fix up code based on pull requests comments. Add test for urls with spaces in them. Hunter Stern 2016-01-28 15:11:49 -08:00

... 13 14 15 16 17 ...