From f478798935db5f238cdf5dc2bcc8cc3e8cf4bb87 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Thu, 6 Jun 2013 16:15:13 -0700 Subject: [PATCH] implement itagPriority and extractLimit parameters --- .../ExtractorYoutubeFormatStream.java | 233 +++++++++++------- .../ExtractorYoutubeFormatStreamTest.java | 131 +++++++--- 2 files changed, 240 insertions(+), 124 deletions(-) diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java index fcc4a994..3fe2dc61 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java @@ -3,8 +3,12 @@ package org.archive.modules.extractor; import java.io.IOException; import java.net.URLDecoder; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; @@ -27,30 +31,42 @@ public class ExtractorYoutubeFormatStream extends Extractor { private static Logger logger = Logger.getLogger(ExtractorYoutubeFormatStream.class.getName()); - /** - * Maximum number of videos to extract - */ { setExtractLimit(1); } public Integer getExtractLimit(){ return (Integer) kp.get("extractLimit"); } + /** + * Maximum number of video urls to extract. A value of 0 means extract all + * discovered video urls. Default is 1. + */ public void setExtractLimit(Integer extractLimit){ kp.put("extractLimit", extractLimit); } - { - setItagPriority(new ArrayList()); + { + setItagPriority(new ArrayList()); } @SuppressWarnings("unchecked") public List getItagPriority() { - return (List) kp.get("itagPriority"); + return (List) kp.get("itagPriority"); } + + /** + * Itag priority list. Youtube itag parameter specifies the video and audio + * format and quality. The default is an empty list, which tells the + * extractor to extract up to extractLimit video urls. When the + * list is not empty, only video urls with itag values in the list are + * extracted. + * + * @see http://en.wikipedia.org/wiki/YouTube + */ public void setItagPriority(List itagPriority) { - kp.put("itagPriority", itagPriority); + kp.put("itagPriority", itagPriority); } - + @Override protected boolean shouldProcess(CrawlURI uri) { return uri.getContentLength() > 0 @@ -59,94 +75,133 @@ public class ExtractorYoutubeFormatStream extends Extractor { uri.getUURI().toCustomString()); } - @Override - protected void extract(CrawlURI uri) { + @Override + protected void extract(CrawlURI uri) { ReplayCharSequence cs; - try { - cs = uri.getRecorder().getContentReplayCharSequence(); - } catch (IOException e) { - uri.getNonFatalFailures().add(e); - logger.log(Level.WARNING,"Failed get of replay char sequence in " + - Thread.currentThread().getName(), e); - return; - } - Matcher matcher = TextUtils.getMatcher("(?is)ytplayer.config = ([^;]*);", cs); - if (matcher.find()) { - String jsonStr = matcher.group(1); - - //logger.fine("Just Extracted: "+jsonStr); - JSONObject json; - try { - json = new JSONObject(jsonStr); - if(json.has("args")){ - JSONObject args = json.getJSONObject("args"); - if(args.has("url_encoded_fmt_stream_map")) { - String stream_map = args.getString("url_encoded_fmt_stream_map"); + try { + cs = uri.getRecorder().getContentReplayCharSequence(); + } catch (IOException e) { + uri.getNonFatalFailures().add(e); + logger.log(Level.WARNING, "Failed get of replay char sequence in " + + Thread.currentThread().getName(), e); + return; + } - //logger.info("Just Extracted: "+stream_map); + Matcher matcher = TextUtils.getMatcher( + "(?is)ytplayer.config = ([^;]*);", cs); + if (matcher.find()) { + String jsonStr = matcher.group(1); - String[] rawVideoList = stream_map.split(","); - LinkedHashMap parsedVideoMap = new LinkedHashMap(); + // logger.fine("Just Extracted: "+jsonStr); + try { + JSONObject json = new JSONObject(jsonStr); + if (json.has("args")) { + JSONObject args = json.getJSONObject("args"); + if (args.has("url_encoded_fmt_stream_map")) { + String streamMap = args.getString("url_encoded_fmt_stream_map"); - //Parse Video Map into itag,url pair - for(int i=0; i < rawVideoList.length; i++) { - String[] videoParams = rawVideoList[i].split("\\u0026"); - String videoURLParam, itagParam, sigParam; - videoURLParam = itagParam = sigParam = ""; + // logger.info("Just Extracted: "+stream_map); + LinkedHashMap parsedVideoMap = parseStreamMap(streamMap); + addPreferredOutlinks(uri, parsedVideoMap); + } + } + } catch (JSONException e) { + logger.log(Level.WARNING, + "Error parsing JSON object - Skipping: " + jsonStr, e); + } + } + TextUtils.recycleMatcher(matcher); + } - for(String param : videoParams){ - - String[] keyValuePair = param.split("="); - if (keyValuePair.length != 2) { - logger.warning("Invalid Video Parameter: " - + param); - continue; - } + // 34 and 35 are most common medium quality flvs, others are in arbitrary order + private static final List DEFAULT_ITAG_PRIORITY = Arrays.asList( + "35", "34", "5", "6", "13", "17", "18", "22", "36", "37", "38", + "43", "44", "45", "46", "82", "83", "84", "85", "100", "101", + "102", "120"); + private static final Set KNOWN_ITAGS = new HashSet(DEFAULT_ITAG_PRIORITY); + + // Add videos as outlinks by priority list + private void addPreferredOutlinks(CrawlURI uri, + LinkedHashMap parsedVideoMap) { + List itagPriority; + if (getItagPriority() != null && !getItagPriority().isEmpty()) { + itagPriority = getItagPriority(); + } else { + itagPriority = DEFAULT_ITAG_PRIORITY; + } + + int extractionCount = 0; + for (String itag : itagPriority) { + if (parsedVideoMap.containsKey(itag) + && (getExtractLimit() <= 0 || extractionCount < getExtractLimit())) { + logger.fine("adding video: " + parsedVideoMap.get(itag)); + addOutlink(uri, parsedVideoMap.get(itag), + org.archive.modules.extractor.LinkContext.EMBED_MISC, + org.archive.modules.extractor.Hop.EMBED); + extractionCount++; + } + } - if (keyValuePair[0].equals("url")) { - videoURLParam = keyValuePair[1]; - } - if (keyValuePair[0].equals("itag")) { - itagParam = keyValuePair[1]; - } - if (keyValuePair[0].equals("sig")) { - sigParam = keyValuePair[1]; - } - } + // if itagPriority not specified, make sure we consider all discovered + // video urls + if (getItagPriority() == null || getItagPriority().isEmpty()) { + Iterator itagKeyIter = parsedVideoMap.keySet().iterator(); + while (itagKeyIter.hasNext() && (getExtractLimit() <= 0 || extractionCount < getExtractLimit())) { + String itag = itagKeyIter.next(); + if (!KNOWN_ITAGS.contains(itag)) { + logger.warning("adding video (with unknown itag " + itag + + "): " + parsedVideoMap.get(itag)); + addOutlink(uri, parsedVideoMap.get(itag), + org.archive.modules.extractor.LinkContext.EMBED_MISC, + org.archive.modules.extractor.Hop.EMBED); + extractionCount++; + } + } + } + } + + private LinkedHashMap parseStreamMap(String streamMap) { + String[] rawVideoList = streamMap.split(","); + LinkedHashMap parsedVideoMap = new LinkedHashMap(); - if(videoURLParam.length()>0 && itagParam.length()>0 && sigParam.length()>0) { - try { - String fixupURL = URLDecoder.decode(videoURLParam+"%26signature="+sigParam, "UTF-8"); - parsedVideoMap.put(itagParam,fixupURL); - } - catch(java.io.UnsupportedEncodingException e) { - logger.warning("Error decoding youtube video URL: "+videoURLParam+"%26signature="+sigParam); - } - } - } - - //Add videos as outlinks by priority list - int extractionCount=0; - for(String itag : getItagPriority()) { - if(parsedVideoMap.containsKey(itag) && extractionCount 0 && itagParam.length() > 0 + && sigParam.length() > 0) { + try { + String fixupURL = URLDecoder.decode(videoURLParam + + "%26signature=" + sigParam, "UTF-8"); + parsedVideoMap.put(itagParam, fixupURL); + } catch (java.io.UnsupportedEncodingException e) { + logger.warning("Error decoding youtube video URL: " + + videoURLParam + "%26signature=" + sigParam); + } + } + } + return parsedVideoMap; + } } \ No newline at end of file diff --git a/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java b/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java index e9ef0fd4..0bd90818 100644 --- a/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java +++ b/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java @@ -5,9 +5,11 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; -import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -36,6 +38,10 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { protected static final String[] EXPECTED_OUTLINKS_SUBSET = { "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=34&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=78D14935180C9DA87E1C562719525D0BB6BE21F9.9BC13425338E5DD6C255EA77136CC424830BCD21" }; + + protected static final String[] EXPECTED_SINGLE_DEFAULT_OUTLINK = { + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=35&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=A23283ED964AA8EF061249CCA6199EDDA6543FF2.89798F8181F2250FCFF24F19B2E59D880D054703", + }; @Override protected Extractor makeExtractor() { @@ -45,32 +51,67 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { return e; } - public void testYoutubeExtractAll() throws Exception { + public void testAllInItagPriority() throws Exception { CrawlURI testUri = createTestUri(TEST_URI); - ArrayList itagPriorityList = new ArrayList(); - itagPriorityList.add("44"); - itagPriorityList.add("35"); - itagPriorityList.add("43"); - itagPriorityList.add("34"); - itagPriorityList.add("18"); - itagPriorityList.add("5"); - itagPriorityList.add("36"); - itagPriorityList.add("17"); - ((ExtractorYoutubeFormatStream)extractor).setItagPriority(itagPriorityList); - ((ExtractorYoutubeFormatStream)extractor).setExtractLimit(10); + List itagPriorityList = Arrays.asList("44", "35", "43", "34", "18", "5", "36", "17"); + extractor().setItagPriority(itagPriorityList); + extractor().setExtractLimit(10); extractor.process(testUri); - HashSet expected = new HashSet(); - for (String expectedLinkString : EXPECTED_OUTLINKS_ALL) { - expected.add(new Link(testUri.getUURI(), - UURIFactory.getInstance(expectedLinkString), - HTMLLinkContext.EMBED_MISC, Hop.EMBED) - ); - } + Set expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_ALL); assertEquals(expected, testUri.getOutLinks()); } + + public void testAllNoPriority() throws Exception { + CrawlURI testUri = createTestUri(TEST_URI); + + extractor().setExtractLimit(0); + extractor.process(testUri); + + Set expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_ALL); + assertEquals(expected, testUri.getOutLinks()); + } + + // test that only itags in the priority list are extracted, even though + // extract limit is large + public void testOnlyInItagPriorityBigLimit() throws Exception { + CrawlURI testUri = createTestUri(TEST_URI); + + List itagPriorityList = Arrays.asList("44", "35", "43"); + extractor().setItagPriority(itagPriorityList); + extractor().setExtractLimit(10); + + extractor.process(testUri); + + assertEquals(3, testUri.getOutLinks().size()); + } + + // test that only itags in the priority list are extracted, even though + // extract limit is unset + public void testOnlyInItagPriorityNoLimit() throws Exception { + CrawlURI testUri = createTestUri(TEST_URI); + + List itagPriorityList = Arrays.asList("44", "35", "43"); + extractor().setItagPriority(itagPriorityList); + extractor().setExtractLimit(0); + + extractor.process(testUri); + + assertEquals(3, testUri.getOutLinks().size()); + } + + + public void testNoPriorityWithLimit() throws InterruptedException, URIException, UnsupportedEncodingException, IOException { + CrawlURI testUri = createTestUri(TEST_URI); + + extractor().setExtractLimit(4); + + extractor.process(testUri); + + assertEquals(4, testUri.getOutLinks().size()); + } public void testDontExtract() throws URIException, UnsupportedEncodingException, IOException, InterruptedException { // not a youtube watch url so shouldProcess() will return false @@ -79,29 +120,48 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { assertEquals(Collections.EMPTY_SET, testUri.getOutLinks()); } - public void testYoutubeExtractSubset() throws Exception { + public void testPriority() throws Exception { CrawlURI testUri = createTestUri(TEST_URI); - - ArrayList itagPriorityList = new ArrayList(); - itagPriorityList.add("37"); - itagPriorityList.add("24"); - itagPriorityList.add("34"); - itagPriorityList.add("35"); - ((ExtractorYoutubeFormatStream)extractor).setItagPriority(itagPriorityList); - ((ExtractorYoutubeFormatStream)extractor).setExtractLimit(1); + + // 37, 24 are not in url_stream_map; 35 appears before 34 in there, but + // with this list we should get 34 + extractor().setItagPriority(Arrays.asList("37", "24", "34", "35")); + extractor().setExtractLimit(1); extractor.process(testUri); - HashSet expected = new HashSet(); - for (String expectedLinkString : EXPECTED_OUTLINKS_SUBSET) { - expected.add(new Link(testUri.getUURI(), - UURIFactory.getInstance(expectedLinkString), + Set expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_SUBSET); + assertEquals(expected, testUri.getOutLinks()); + } + + public void testDefaultItag() throws URIException, UnsupportedEncodingException, IOException, InterruptedException { + CrawlURI testUri = createTestUri(TEST_URI); + + extractor().setExtractLimit(1); + + assertEquals(Collections.EMPTY_LIST, extractor().getItagPriority()); + + extractor.process(testUri); + + Set expected = makeLinkSet(testUri, EXPECTED_SINGLE_DEFAULT_OUTLINK); + assertEquals(expected, testUri.getOutLinks()); + } + + private Set makeLinkSet(CrawlURI sourceUri, String[] urlStrs) throws URIException { + HashSet linkSet = new HashSet(); + for (String urlStr : urlStrs) { + linkSet.add(new Link(sourceUri.getUURI(), + UURIFactory.getInstance(urlStr), HTMLLinkContext.EMBED_MISC, Hop.EMBED) ); } - assertEquals(expected, testUri.getOutLinks()); + return linkSet; + } - } + private ExtractorYoutubeFormatStream extractor() { + return (ExtractorYoutubeFormatStream)extractor; + } + private CrawlURI createTestUri(String urlStr) throws URIException, UnsupportedEncodingException, IOException { UURI testUuri = UURIFactory.getInstance(urlStr); @@ -124,6 +184,7 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { return testUri; } } + class UnitTestUriLoggerModule implements UriErrorLoggerModule { final private static Logger LOGGER = Logger.getLogger(UnitTestUriLoggerModule.class.getName());