From feec423b52adf2e88a98f32e64bd92178cfe65d6 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 5 Jun 2013 13:16:55 -0700 Subject: [PATCH] import Adam's ExtractorYoutubeFormatStream from its former home https://github.com/adam-miller/ExtractorYoutubeFormatStream --- .../ExtractorYoutubeFormatStream.java | 162 ++++++++++++++++++ .../ExtractorYoutubeFormatStreamTest.java | 112 ++++++++++++ 2 files changed, 274 insertions(+) create mode 100644 contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java create mode 100644 contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java new file mode 100644 index 00000000..0e6fa8b0 --- /dev/null +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java @@ -0,0 +1,162 @@ +package org.archive.modules.extractor; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.Iterator; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; + +import org.apache.commons.httpclient.URIException; +import org.archive.io.ReplayCharSequence; +import org.archive.modules.CrawlMetadata; +import org.archive.modules.CrawlURI; +import org.archive.modules.net.RobotsPolicy; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.archive.util.DevUtils; +import org.archive.util.TextUtils; +import org.archive.util.UriUtils; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +/** + * Youtube stream URI extractor. + * This will check the content of the youtube watch page looking for the url_encoded_fmt_stream_map json value. + * The json object is decoded and the stream URIs are constructed and queued. + * + * @contributor adam + * + */ +public class ExtractorYoutubeFormatStream extends ContentExtractor { + private static Logger logger = + Logger.getLogger(ExtractorYoutubeFormatStream.class.getName()); + + /** + * Maximum number of videos to extract + */ + { + setExtractLimit(1); + } + public Integer getExtractLimit(){ + return (Integer) kp.get("extractLimit"); + } + public void setExtractLimit(Integer extractLimit){ + kp.put("extractLimit", extractLimit); + } + + { + setItagPriority(new ArrayList()); + } + @SuppressWarnings("unchecked") + public List getItagPriority() { + return (List) kp.get("itagPriority"); + } + public void setItagPriority(List itagPriority) { + kp.put("itagPriority", itagPriority); + } + + @Override + protected boolean shouldExtract(CrawlURI uri){ + return true; + // if (uri.getFetchStatus() != 200) { + // //verify we are a youtube watch page + // return false; + // } + // else { + // return true; + // } + } + @Override + protected boolean innerExtract(CrawlURI uri) { + ReplayCharSequence cs; + try { + cs = uri.getRecorder().getContentReplayCharSequence(); + } catch (IOException e) { + uri.getNonFatalFailures().add(e); + logger.log(Level.WARNING,"Failed get of replay char sequence in " + + Thread.currentThread().getName(), e); + return false; + } + Matcher matcher = TextUtils.getMatcher("(?is)ytplayer.config = ([^;]*);", cs); + if (matcher.find()) { + String jsonStr = matcher.group(1); + + logger.info("Just Extracted: "+jsonStr); + JSONObject json; + try { + json = new JSONObject(jsonStr); + if(json.has("args")){ + JSONObject args = json.getJSONObject("args"); + if(args.has("url_encoded_fmt_stream_map")) { + String stream_map = args.getString("url_encoded_fmt_stream_map"); + + logger.info("Just Extracted: "+stream_map); + + String[] videos = stream_map.split(","); + + for(int i=0; i < videos.length; i++) { + String[] videoParams = videos[i].split("\u0026"); + + Matcher videoMatchPre = TextUtils.getMatcher("(?is)sig=([^&]*).*url=(http[^&]*)", videos[i]); + Matcher videoMatchPost = TextUtils.getMatcher("(?is)url=(http[^&]*).*sig=([^&]*)", videos[i]); + String videoUri=""; + String prefix,suffix; + + if(videoMatchPre.find()) { + prefix = videoMatchPre.group(2); + suffix = videoMatchPre.group(1); + } + else if(videoMatchPost.find()) { + prefix = videoMatchPost.group(1); + suffix = videoMatchPost.group(2); + } + else + return false; + + try { + videoUri = new java.net.URI(prefix+"%26signature="+suffix).getPath(); + } + catch(java.net.URISyntaxException e) { + logger.warning("problem decoding link " + prefix+"%26signature="+suffix + " - " + e); + } + + TextUtils.recycleMatcher(videoMatchPre); + TextUtils.recycleMatcher(videoMatchPost); + + + logger.info("found video: "+videoUri); + int max = getExtractorParameters().getMaxOutlinks(); + String gen204 = videoUri.substring(0,videoUri.lastIndexOf("/"))+"/generate_204"; + + addOutlink(uri,gen204.toString(), org.archive.modules.extractor.LinkContext.EMBED_MISC, org.archive.modules.extractor.Hop.EMBED); + + logger.warning("creating gen204: "+gen204); + logger.warning("newOutlinkCount: "+numberOfLinksExtracted.incrementAndGet()); + + addOutlink(uri,videoUri.toString(), org.archive.modules.extractor.LinkContext.EMBED_MISC, org.archive.modules.extractor.Hop.EMBED); + + logger.warning("adding video: "+videoUri); + logger.warning("newOutlinkCount: "+numberOfLinksExtracted.incrementAndGet()); + + } + } + } + } + catch(JSONException e) { + logger.log(Level.WARNING, "Error parsing JSON object - Skipping: "+jsonStr, e); + } + } + TextUtils.recycleMatcher(matcher); + return true; + } + + +} \ No newline at end of file diff --git a/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java b/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java new file mode 100644 index 00000000..da7d89bb --- /dev/null +++ b/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java @@ -0,0 +1,112 @@ +package org.archive.modules.extractor; + +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.httpclient.URIException; +import org.archive.modules.CrawlURI; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.archive.util.Recorder; + + +public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { + + protected static final String TEST_URI = "http://www.youtube.com/watch?v=_BFJN62hZp0"; + protected static final String[] EXPECTED_OUTLINKS = { + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?key=yt1&ip=208.70.31.237&sver=3&mv=m&itag=44&source=youtube&fexp=911418%2C932402%2C923500%2C901472%2C900352%2C924605%2C928201%2C901208%2C929123%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C932802%2C928017%2C912512%2C912518%2C911416%2C906906%2C904476%2C930807%2C919373%2C906836%2C933701%2C926403%2C912711%2C929606%2C910075&expire=1370478870&newshard=yes&ms=au&upn=ko-PmxWJzEg&ratebypass=yes&id=fc114937ada1669d&mt=1370455230&sparams=cp%2Cid%2Cip%2Cipbits%2Citag%2Cratebypass%2Csource%2Cupn%2Cexpire&ipbits=8&cp=U0hWRVRSVl9MSkNONl9MTFpJOm8tNWxnVEptdFY2&signature=097BABD561F197C107AE9BE57C3CE4F2A9193C57.25249074A61F72F639C4635C31DA48E2D24F125F", + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?algorithm=throttle-factor&ip=208.70.31.237&sver=3&factor=1.25&mt=1370455230&itag=35&source=youtube&fexp=911418%2C932402%2C923500%2C901472%2C900352%2C924605%2C928201%2C901208%2C929123%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C932802%2C928017%2C912512%2C912518%2C911416%2C906906%2C904476%2C930807%2C919373%2C906836%2C933701%2C926403%2C912711%2C929606%2C910075&expire=1370478870&newshard=yes&ms=au&upn=ko-PmxWJzEg&id=fc114937ada1669d&burst=40&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&mv=m&ipbits=8&key=yt1&cp=U0hWRVRSVl9MSkNONl9MTFpJOm8tNWxnVEptdFY2&signature=745286D6E79C965AE1AFC046A2348778F3DBD7E1.35BF2F520B7A42CADF2F076C72F3943C38CB85A8", + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?key=yt1&ip=208.70.31.237&sver=3&mv=m&itag=43&source=youtube&fexp=911418%2C932402%2C923500%2C901472%2C900352%2C924605%2C928201%2C901208%2C929123%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C932802%2C928017%2C912512%2C912518%2C911416%2C906906%2C904476%2C930807%2C919373%2C906836%2C933701%2C926403%2C912711%2C929606%2C910075&expire=1370478870&newshard=yes&ms=au&upn=ko-PmxWJzEg&ratebypass=yes&id=fc114937ada1669d&mt=1370455230&sparams=cp%2Cid%2Cip%2Cipbits%2Citag%2Cratebypass%2Csource%2Cupn%2Cexpire&ipbits=8&cp=U0hWRVRSVl9MSkNONl9MTFpJOm8tNWxnVEptdFY2&signature=AC8D17DAF1D2553C15F4E23D24D1C7013A0637FB.596D6011AB9AA4E6025F7784E204BD6BD2FC9408", + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?algorithm=throttle-factor&ip=208.70.31.237&sver=3&factor=1.25&mt=1370455230&itag=34&source=youtube&fexp=911418%2C932402%2C923500%2C901472%2C900352%2C924605%2C928201%2C901208%2C929123%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C932802%2C928017%2C912512%2C912518%2C911416%2C906906%2C904476%2C930807%2C919373%2C906836%2C933701%2C926403%2C912711%2C929606%2C910075&expire=1370478870&newshard=yes&ms=au&upn=ko-PmxWJzEg&id=fc114937ada1669d&burst=40&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&mv=m&ipbits=8&key=yt1&cp=U0hWRVRSVl9MSkNONl9MTFpJOm8tNWxnVEptdFY2&signature=4A6FEC2DB96926893C2359E8ED680A1E5B21B71C.74B3D1DC45C2F1DA3CAFB4218704AD2A4B61D4C2", + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?key=yt1&ip=208.70.31.237&sver=3&mv=m&itag=18&source=youtube&fexp=911418%2C932402%2C923500%2C901472%2C900352%2C924605%2C928201%2C901208%2C929123%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C932802%2C928017%2C912512%2C912518%2C911416%2C906906%2C904476%2C930807%2C919373%2C906836%2C933701%2C926403%2C912711%2C929606%2C910075&expire=1370478870&newshard=yes&ms=au&upn=ko-PmxWJzEg&ratebypass=yes&id=fc114937ada1669d&mt=1370455230&sparams=cp%2Cid%2Cip%2Cipbits%2Citag%2Cratebypass%2Csource%2Cupn%2Cexpire&ipbits=8&cp=U0hWRVRSVl9MSkNONl9MTFpJOm8tNWxnVEptdFY2&signature=D2B301B46D87D1980C65FF9FEF67BD77CAC05F0F.5C76D533CC6B3CB06E114E9A97D4F62710EFC102", + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?algorithm=throttle-factor&ip=208.70.31.237&sver=3&factor=1.25&mt=1370455230&itag=5&source=youtube&fexp=911418%2C932402%2C923500%2C901472%2C900352%2C924605%2C928201%2C901208%2C929123%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C932802%2C928017%2C912512%2C912518%2C911416%2C906906%2C904476%2C930807%2C919373%2C906836%2C933701%2C926403%2C912711%2C929606%2C910075&expire=1370478870&newshard=yes&ms=au&upn=ko-PmxWJzEg&id=fc114937ada1669d&burst=40&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&mv=m&ipbits=8&key=yt1&cp=U0hWRVRSVl9MSkNONl9MTFpJOm8tNWxnVEptdFY2&signature=6BBDDD158B7FDA10F5ECE1CBB5F784AF7BB341E0.4387402752DF53E2D1C5185503F6C0D96A4BBB5F", + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?algorithm=throttle-factor&ip=208.70.31.237&sver=3&factor=1.25&mt=1370455230&itag=36&source=youtube&fexp=911418%2C932402%2C923500%2C901472%2C900352%2C924605%2C928201%2C901208%2C929123%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C932802%2C928017%2C912512%2C912518%2C911416%2C906906%2C904476%2C930807%2C919373%2C906836%2C933701%2C926403%2C912711%2C929606%2C910075&expire=1370478870&newshard=yes&ms=au&upn=ko-PmxWJzEg&id=fc114937ada1669d&burst=40&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&mv=m&ipbits=8&key=yt1&cp=U0hWRVRSVl9MSkNONl9MTFpJOm8tNWxnVEptdFY2&signature=7C7DF993A78D3CA3D1CB80340E8EB52EFE9084A5.12F708A4B708EC0290B697F54079B31CF067B83C", + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?algorithm=throttle-factor&ip=208.70.31.237&sver=3&factor=1.25&mt=1370455230&itag=17&source=youtube&fexp=911418%2C932402%2C923500%2C901472%2C900352%2C924605%2C928201%2C901208%2C929123%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C932802%2C928017%2C912512%2C912518%2C911416%2C906906%2C904476%2C930807%2C919373%2C906836%2C933701%2C926403%2C912711%2C929606%2C910075&expire=1370478870&newshard=yes&ms=au&upn=ko-PmxWJzEg&id=fc114937ada1669d&burst=40&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&mv=m&ipbits=8&key=yt1&cp=U0hWRVRSVl9MSkNONl9MTFpJOm8tNWxnVEptdFY2&signature=787424AE0AA71635806342CDEDF148D340601AF5.329DEB777CDF474FEA3B4D4A041F5188FA892981" + + }; + + protected static final String[] TEST_CONTENT_CHUNKS = { + +"" + + }; + + @Override + protected Extractor makeExtractor() { + ExtractorYoutubeFormatStream e = new ExtractorYoutubeFormatStream(); + UriErrorLoggerModule ulm = new UnitTestUriLoggerModule(); + e.setLoggerModule(ulm); + + ArrayList itagPriorityList = new ArrayList(); + itagPriorityList.add("37"); + itagPriorityList.add("24"); + itagPriorityList.add("35"); + itagPriorityList.add("34"); + + e.setItagPriority(itagPriorityList); + e.setExtractLimit(8); + + return e; + } + public void testYoutubeExtractExample() throws Exception { + UURI testUuri = UURIFactory.getInstance(TEST_URI); + CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC); + StringBuilder content = new StringBuilder(); + for (String chunk: TEST_CONTENT_CHUNKS) { + content.append(chunk); + } + Recorder recorder = createRecorder(content.toString(), "UTF-8"); + testUri.setContentType("text/html"); + //set to 200 + testUri.setRecorder(recorder); + testUri.setContentSize(content.length()); + + extractor.process(testUri); + + for (String expectedLinkString: EXPECTED_OUTLINKS) { + Link expectedLink = new Link(testUri.getUURI(), + UURIFactory.getInstance(expectedLinkString), + HTMLLinkContext.EMBED_MISC, Hop.EMBED); + assertTrue(testUri.getOutLinks().contains(expectedLink)); + } + } +} +class UnitTestUriLoggerModule implements UriErrorLoggerModule { + private static final long serialVersionUID = 1L; + + final private static Logger LOGGER = + Logger.getLogger(UnitTestUriLoggerModule.class.getName()); + + public void info(String info) { + LOGGER.log(Level.INFO, info); + System.out.println("INFO - "+info); + } + public void logUriError(URIException e, UURI u, CharSequence l) { + LOGGER.log(Level.INFO, u.toString(), e); + System.out.println("Info - "+ u.toString()+" "+e.toString()); + } + + public Logger getLogger(String name) { + return LOGGER; + } + + + +} \ No newline at end of file