diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java index 3fe2dc61..cab71424 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java @@ -88,7 +88,7 @@ public class ExtractorYoutubeFormatStream extends Extractor { } Matcher matcher = TextUtils.getMatcher( - "(?is)ytplayer.config = ([^;]*);", cs); + "(?is)ytplayer\\.config = (\\{.*?\\})(;||$)", cs); if (matcher.find()) { String jsonStr = matcher.group(1); diff --git a/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java b/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java index 0bd90818..335be6f2 100644 --- a/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java +++ b/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java @@ -24,6 +24,8 @@ import org.archive.util.Recorder; public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { protected static final String TEST_URI = "http://www.youtube.com/watch?v=_BFJN62hZp0"; + protected static final String TEST_RESOURCE_FILE_NAME = "ExtractorYoutubeFormatStream.txt"; + protected static final String[] EXPECTED_OUTLINKS_ALL = { "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=44&mt=1370471490&source=youtube&mv=m&sver=3&ratebypass=yes&ms=au&sparams=cp%2Cid%2Cip%2Cipbits%2Citag%2Cratebypass%2Csource%2Cupn%2Cexpire&ipbits=8&expire=1370493270&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&id=fc114937ada1669d&upn=t-LMF5MC9BA&signature=98D0AC4D1B545DE6D5C531A5DB7902877511632A.5700D588C7659DE8A8621EC5110CB638D0EF4A39", "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=35&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=A23283ED964AA8EF061249CCA6199EDDA6543FF2.89798F8181F2250FCFF24F19B2E59D880D054703", @@ -52,7 +54,7 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { } public void testAllInItagPriority() throws Exception { - CrawlURI testUri = createTestUri(TEST_URI); + CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME); List itagPriorityList = Arrays.asList("44", "35", "43", "34", "18", "5", "36", "17"); extractor().setItagPriority(itagPriorityList); @@ -65,7 +67,7 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { } public void testAllNoPriority() throws Exception { - CrawlURI testUri = createTestUri(TEST_URI); + CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME); extractor().setExtractLimit(0); extractor.process(testUri); @@ -77,7 +79,7 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { // test that only itags in the priority list are extracted, even though // extract limit is large public void testOnlyInItagPriorityBigLimit() throws Exception { - CrawlURI testUri = createTestUri(TEST_URI); + CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME); List itagPriorityList = Arrays.asList("44", "35", "43"); extractor().setItagPriority(itagPriorityList); @@ -91,7 +93,7 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { // test that only itags in the priority list are extracted, even though // extract limit is unset public void testOnlyInItagPriorityNoLimit() throws Exception { - CrawlURI testUri = createTestUri(TEST_URI); + CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME); List itagPriorityList = Arrays.asList("44", "35", "43"); extractor().setItagPriority(itagPriorityList); @@ -104,7 +106,7 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { public void testNoPriorityWithLimit() throws InterruptedException, URIException, UnsupportedEncodingException, IOException { - CrawlURI testUri = createTestUri(TEST_URI); + CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME); extractor().setExtractLimit(4); @@ -115,13 +117,13 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { public void testDontExtract() throws URIException, UnsupportedEncodingException, IOException, InterruptedException { // not a youtube watch url so shouldProcess() will return false - CrawlURI testUri = createTestUri("http://archive.org/watch?w=blah"); + CrawlURI testUri = createTestUri("http://archive.org/watch?w=blah", TEST_RESOURCE_FILE_NAME); extractor.process(testUri); assertEquals(Collections.EMPTY_SET, testUri.getOutLinks()); } public void testPriority() throws Exception { - CrawlURI testUri = createTestUri(TEST_URI); + CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME); // 37, 24 are not in url_stream_map; 35 appears before 34 in there, but // with this list we should get 34 @@ -133,9 +135,17 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { Set expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_SUBSET); assertEquals(expected, testUri.getOutLinks()); } + + public void testAlternatePage() throws Exception { + CrawlURI testUri = createTestUri("http://www.youtube.com/watch?v=OyJ3CafAM1Q","ExtractorYoutubeFormatStream2.txt"); + extractor().setExtractLimit(0); + extractor.process(testUri); + + assertEquals(true, testUri.getOutLinks().size()>0); + } public void testDefaultItag() throws URIException, UnsupportedEncodingException, IOException, InterruptedException { - CrawlURI testUri = createTestUri(TEST_URI); + CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME); extractor().setExtractLimit(1); @@ -162,12 +172,12 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { return (ExtractorYoutubeFormatStream)extractor; } - private CrawlURI createTestUri(String urlStr) throws URIException, + private CrawlURI createTestUri(String urlStr, String resourceFileName) throws URIException, UnsupportedEncodingException, IOException { UURI testUuri = UURIFactory.getInstance(urlStr); CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC); - InputStream is = ExtractorYoutubeFormatStreamTest.class.getClassLoader().getResourceAsStream("ExtractorYoutubeFormatStream.txt"); + InputStream is = ExtractorYoutubeFormatStreamTest.class.getClassLoader().getResourceAsStream(resourceFileName); BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8")); StringBuilder content = new StringBuilder(); String line = ""; diff --git a/contrib/src/test/resources/ExtractorYoutubeFormatStream2.txt b/contrib/src/test/resources/ExtractorYoutubeFormatStream2.txt new file mode 100644 index 00000000..8e69900d --- /dev/null +++ b/contrib/src/test/resources/ExtractorYoutubeFormatStream2.txt @@ -0,0 +1,2445 @@ + + + + Florida's Special Places: Circle B Bar Reserve in Lakeland - YouTube + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
Upload
+ +
+ + + +
+ + + + +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+
+ +
+

+ This video is unavailable. +

+
+
+
+ + +
+ + + +
+ + + + + + + +
+ +
+ +
+ +
+
+
+
+
+

+ + + + + + + + Florida's Special Places: Circle B Bar Reserve in Lakeland + + +

+
+ +
+ + + + Audubon Florida + + + + + +Audubon Florida·34 videos
92 + +
+ +
+
+ +
+
+
+

+ Subscription preferences + + +

+
+
+
+
Loading...
+
+ +
+
+
+
+ + +
+
+
+
+ +
+
+
+
+
+
+
Working...
+
+
+ +
+
+
+
+
+ + +
+ +
+ 1,184 + + +
+
+
+
+ + Like + +     Dislike + 0 + + + +
+
+
+
+

Sign in to YouTube

+
+ Sign in with your Google Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to like Audubon Florida's video. + +
+
    +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
+ +
+
+

Sign in to YouTube

+
+ Sign in with your Google Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to dislike Audubon Florida's video. + +
+
    +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
+ +
+ +
+
+ + + + + + + + + + +
+

Sign in to YouTube

+
+ Sign in with your Google Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to add Audubon Florida's video to your playlist. + +
+
    +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
+ +
+ + + + + + + + + + + + + +
+
+ +
+
+
+
+
+

+ Uploaded on Dec 9, 2011 + +

+
+

Lake Region Audubon Society has nominated Circle B Bar Reserve in Lakeland as one of Florida's Special Places. You can nominate your favorite natural Florida spots: http://www.FloridasSpecialPlaces.org

+
+
+
    +
  • +

    +Category +

    +
    +

    Nonprofits & Activism

    + +
    +
    • + + +
  • +

    License

    +
    +

    +Standard YouTube License +

    + +
    • + +
+
+ +
+
    + +
+
+ +
+
+ +
+
+ +
+
+
+
+ +
+
+
+ + +
+
+
+ +
+ +
+
+ + +
+
+ +
+
+ + +
+
+ +
+
+ + +
+
+ +
+ +
+ +
+
+ Ratings have been disabled for this video. +
+ +
+ +
+
+ Rating is available when the video has been rented. +
+ +
+ +
+
+ This feature is not available right now. Please try again later. +
+
+ + +
+ +
+
+
+ + +
+ +
+

+ + All Comments (5) + + +

+ + +
+ Sign in now to post a comment! + +
+ + +
    + + + +
  • + + + + + + KazokuMugiwara + + + + + + + + + +
    +

    + + KazokuMugiwara + + + + 2 months ago + + +

    + + +
    +

    Woodstock!!

    +

    I'm sooo excited.

    + +
    + +
    + + · + + +
    +

    Sign in to YouTube

    +
    + Sign in with your YouTube Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to rate KazokuMugiwara's comment. + +
    +
      +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    + +
    +
    +

    Sign in to YouTube

    +
    + Sign in with your YouTube Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to rate KazokuMugiwara's comment. + +
    +
      +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    + +
    +     +
    + +
    + + +
    • + + + + +
  • + + + + + + Thomas Bussell + + + + + + + + + +
    +

    + + Thomas Bussell + + + + 4 months ago + + +

    + + +
    +

    Nice!!

    +

    

    + +
    + +
    + + · + + +
    +

    Sign in to YouTube

    +
    + Sign in with your YouTube Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to rate Thomas Bussell's comment. + +
    +
      +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    + +
    +
    +

    Sign in to YouTube

    +
    + Sign in with your YouTube Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to rate Thomas Bussell's comment. + +
    +
      +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    + +
    +     +
    + +
    + + +
    • + + + + +
  • + + + + + + l67swap1 + + + + + + + + + +
    +

    + + l67swap1 + + + + 10 months ago + + +

    + + +
    +

    Love this place ... amazing video

    + +
    + +
    + + · + + +
    +

    Sign in to YouTube

    +
    + Sign in with your YouTube Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to rate l67swap1's comment. + +
    +
      +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    + +
    +
    +

    Sign in to YouTube

    +
    + Sign in with your YouTube Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to rate l67swap1's comment. + +
    +
      +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    + +
    +     +
    + +
    + + +
    • + + + + +
  • + + + + + + mflynn2009 + + + + + + + + + +
    +

    + + mflynn2009 + + + + 1 year ago + + +

    + + +
    +

    You captured it perfect!!

    + +
    + +
    + + · + + +
    +

    Sign in to YouTube

    +
    + Sign in with your YouTube Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to rate mflynn2009's comment. + +
    +
      +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    + +
    +
    +

    Sign in to YouTube

    +
    + Sign in with your YouTube Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to rate mflynn2009's comment. + +
    +
      +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    + +
    +     +
    + +
    + + +
    • + + + + +
  • + + + + + + MattUNI2005 + + + + + + + + + +
    +

    + + MattUNI2005 + + + + 1 year ago + + +

    + + +
    +

    One of my all-time favorite spots in the world - Circle B Bar is simply amazing - great video!

    + +
    + +
    + + · + + +
    +

    Sign in to YouTube

    +
    + Sign in with your YouTube Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to rate MattUNI2005's comment. + +
    +
      +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    + +
    +
    +

    Sign in to YouTube

    +
    + Sign in with your YouTube Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to rate MattUNI2005's comment. + +
    +
      +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    + +
    +     +
    + +
    + + +
    • + +
+ +
+ + + + +
    +
  • Loading comment...
    • +
+
+
Loading...
+ + + +
+ + + +
+
+ + + +
+
+Advertisement +
+
+
+
+ + +
+ +
+ +
+ + +
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+
Loading...
+
+ +
+
+
+
+ +
+
+
+
+
+
+
Working...
+
+
+ +
+
+
+
+
+ + + +
+ to add this to Watch Later + +
+ + + + + + + + + +
+