lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-22 13:46:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

implement itagPriority and extractLimit parameters

Browse Source
This commit is contained in:
Noah Levitt
2013-06-06 16:15:13 -07:00
parent 593cae448e
commit f478798935
2 changed files with 240 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java
+144 -89
View File
@@ -3,8 +3,12 @@ package org.archive.modules.extractor;
import java.io.IOException;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
@@ -27,30 +31,42 @@ public class ExtractorYoutubeFormatStream extends Extractor {
private static Logger logger =
Logger.getLogger(ExtractorYoutubeFormatStream.class.getName());
/**
* Maximum number of videos to extract
*/
{
setExtractLimit(1);
}
public Integer getExtractLimit(){
return (Integer) kp.get("extractLimit");
}
/**
* Maximum number of video urls to extract. A value of 0 means extract all
* discovered video urls. Default is 1.
*/
public void setExtractLimit(Integer extractLimit){
kp.put("extractLimit", extractLimit);
}
{
setItagPriority(new ArrayList<String>());
{
setItagPriority(new ArrayList<String>());
}
@SuppressWarnings("unchecked")
public List<String> getItagPriority() {
return (List<String>) kp.get("itagPriority");
return (List<String>) kp.get("itagPriority");
}
/**
* Itag priority list. Youtube itag parameter specifies the video and audio
* format and quality. The default is an empty list, which tells the
* extractor to extract up to extractLimit video urls. When the
* list is not empty, only video urls with itag values in the list are
* extracted.
*
* @see <a
* href="http://en.wikipedia.org/wiki/YouTube">http://en.wikipedia.org/wiki/YouTube</a>
*/
public void setItagPriority(List<String> itagPriority) {
kp.put("itagPriority", itagPriority);
kp.put("itagPriority", itagPriority);
}
@Override
protected boolean shouldProcess(CrawlURI uri) {
return uri.getContentLength() > 0
@@ -59,94 +75,133 @@ public class ExtractorYoutubeFormatStream extends Extractor {
uri.getUURI().toCustomString());
}
@Override
protected void extract(CrawlURI uri) {
@Override
protected void extract(CrawlURI uri) {
ReplayCharSequence cs;
try {
cs = uri.getRecorder().getContentReplayCharSequence();
} catch (IOException e) {
uri.getNonFatalFailures().add(e);
logger.log(Level.WARNING,"Failed get of replay char sequence in " +
Thread.currentThread().getName(), e);
return;
}
Matcher matcher = TextUtils.getMatcher("(?is)ytplayer.config = ([^;]*);", cs);
if (matcher.find()) {
String jsonStr = matcher.group(1);
//logger.fine("Just Extracted: "+jsonStr);
JSONObject json;
try {
json = new JSONObject(jsonStr);
if(json.has("args")){
JSONObject args = json.getJSONObject("args");
if(args.has("url_encoded_fmt_stream_map")) {
String stream_map = args.getString("url_encoded_fmt_stream_map");
try {
cs = uri.getRecorder().getContentReplayCharSequence();
} catch (IOException e) {
uri.getNonFatalFailures().add(e);
logger.log(Level.WARNING, "Failed get of replay char sequence in "
+ Thread.currentThread().getName(), e);
return;
}
//logger.info("Just Extracted: "+stream_map);
Matcher matcher = TextUtils.getMatcher(
"(?is)ytplayer.config = ([^;]*);", cs);
if (matcher.find()) {
String jsonStr = matcher.group(1);
String[] rawVideoList = stream_map.split(",");
LinkedHashMap<String,String> parsedVideoMap = new LinkedHashMap<String,String>();
// logger.fine("Just Extracted: "+jsonStr);
try {
JSONObject json = new JSONObject(jsonStr);
if (json.has("args")) {
JSONObject args = json.getJSONObject("args");
if (args.has("url_encoded_fmt_stream_map")) {
String streamMap = args.getString("url_encoded_fmt_stream_map");
//Parse Video Map into itag,url pair
for(int i=0; i < rawVideoList.length; i++) {
String[] videoParams = rawVideoList[i].split("\\u0026");
String videoURLParam, itagParam, sigParam;
videoURLParam = itagParam = sigParam = "";
// logger.info("Just Extracted: "+stream_map);
LinkedHashMap<String, String> parsedVideoMap = parseStreamMap(streamMap);
addPreferredOutlinks(uri, parsedVideoMap);
}
}
} catch (JSONException e) {
logger.log(Level.WARNING,
"Error parsing JSON object - Skipping: " + jsonStr, e);
}
}
TextUtils.recycleMatcher(matcher);
}
for(String param : videoParams){
String[] keyValuePair = param.split("=");
if (keyValuePair.length != 2) {
logger.warning("Invalid Video Parameter: "
+ param);
continue;
}
// 34 and 35 are most common medium quality flvs, others are in arbitrary order
private static final List<String> DEFAULT_ITAG_PRIORITY = Arrays.asList(
"35", "34", "5", "6", "13", "17", "18", "22", "36", "37", "38",
"43", "44", "45", "46", "82", "83", "84", "85", "100", "101",
"102", "120");
private static final Set<String> KNOWN_ITAGS = new HashSet<String>(DEFAULT_ITAG_PRIORITY);
// Add videos as outlinks by priority list
private void addPreferredOutlinks(CrawlURI uri,
LinkedHashMap<String, String> parsedVideoMap) {
List<String> itagPriority;
if (getItagPriority() != null && !getItagPriority().isEmpty()) {
itagPriority = getItagPriority();
} else {
itagPriority = DEFAULT_ITAG_PRIORITY;
}
int extractionCount = 0;
for (String itag : itagPriority) {
if (parsedVideoMap.containsKey(itag)
&& (getExtractLimit() <= 0 || extractionCount < getExtractLimit())) {
logger.fine("adding video: " + parsedVideoMap.get(itag));
addOutlink(uri, parsedVideoMap.get(itag),
org.archive.modules.extractor.LinkContext.EMBED_MISC,
org.archive.modules.extractor.Hop.EMBED);
extractionCount++;
}
}
if (keyValuePair[0].equals("url")) {
videoURLParam = keyValuePair[1];
}
if (keyValuePair[0].equals("itag")) {
itagParam = keyValuePair[1];
}
if (keyValuePair[0].equals("sig")) {
sigParam = keyValuePair[1];
}
}
// if itagPriority not specified, make sure we consider all discovered
// video urls
if (getItagPriority() == null || getItagPriority().isEmpty()) {
Iterator<String> itagKeyIter = parsedVideoMap.keySet().iterator();
while (itagKeyIter.hasNext() && (getExtractLimit() <= 0 || extractionCount < getExtractLimit())) {
String itag = itagKeyIter.next();
if (!KNOWN_ITAGS.contains(itag)) {
logger.warning("adding video (with unknown itag " + itag
+ "): " + parsedVideoMap.get(itag));
addOutlink(uri, parsedVideoMap.get(itag),
org.archive.modules.extractor.LinkContext.EMBED_MISC,
org.archive.modules.extractor.Hop.EMBED);
extractionCount++;
}
}
}
}
private LinkedHashMap<String, String> parseStreamMap(String streamMap) {
String[] rawVideoList = streamMap.split(",");
LinkedHashMap<String, String> parsedVideoMap = new LinkedHashMap<String, String>();
if(videoURLParam.length()>0 && itagParam.length()>0 && sigParam.length()>0) {
try {
String fixupURL = URLDecoder.decode(videoURLParam+"%26signature="+sigParam, "UTF-8");
parsedVideoMap.put(itagParam,fixupURL);
}
catch(java.io.UnsupportedEncodingException e) {
logger.warning("Error decoding youtube video URL: "+videoURLParam+"%26signature="+sigParam);
}
}
}
//Add videos as outlinks by priority list
int extractionCount=0;
for(String itag : getItagPriority()) {
if(parsedVideoMap.containsKey(itag) && extractionCount<getExtractLimit()) {
int hostnameEndIndex = parsedVideoMap.get(itag).lastIndexOf("/");
if(hostnameEndIndex<1) {
continue;
}
logger.warning("adding video: "+parsedVideoMap.get(itag));
addOutlink(uri,parsedVideoMap.get(itag), org.archive.modules.extractor.LinkContext.EMBED_MISC, org.archive.modules.extractor.Hop.EMBED);
extractionCount++;
}
}
}
}
}
catch(JSONException e) {
logger.log(Level.WARNING, "Error parsing JSON object - Skipping: "+jsonStr, e);
}
}
TextUtils.recycleMatcher(matcher);
}
// Parse Video Map into itag,url pair
for (int i = 0; i < rawVideoList.length; i++) {
String[] videoParams = rawVideoList[i].split("\\u0026");
String videoURLParam, itagParam, sigParam;
videoURLParam = itagParam = sigParam = "";
for (String param : videoParams) {
String[] keyValuePair = param.split("=");
if (keyValuePair.length != 2) {
logger.warning("Invalid Video Parameter: " + param);
continue;
}
if (keyValuePair[0].equals("url")) {
videoURLParam = keyValuePair[1];
}
if (keyValuePair[0].equals("itag")) {
itagParam = keyValuePair[1];
}
if (keyValuePair[0].equals("sig")) {
sigParam = keyValuePair[1];
}
}
if (videoURLParam.length() > 0 && itagParam.length() > 0
&& sigParam.length() > 0) {
try {
String fixupURL = URLDecoder.decode(videoURLParam
+ "%26signature=" + sigParam, "UTF-8");
parsedVideoMap.put(itagParam, fixupURL);
} catch (java.io.UnsupportedEncodingException e) {
logger.warning("Error decoding youtube video URL: "
+ videoURLParam + "%26signature=" + sigParam);
}
}
}
return parsedVideoMap;
}
}
contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java
+96 -35
View File
@@ -5,9 +5,11 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -36,6 +38,10 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase {
protected static final String[] EXPECTED_OUTLINKS_SUBSET = {
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=34&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=78D14935180C9DA87E1C562719525D0BB6BE21F9.9BC13425338E5DD6C255EA77136CC424830BCD21"
};
protected static final String[] EXPECTED_SINGLE_DEFAULT_OUTLINK = {
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=35&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=A23283ED964AA8EF061249CCA6199EDDA6543FF2.89798F8181F2250FCFF24F19B2E59D880D054703",
};
@Override
protected Extractor makeExtractor() {
@@ -45,32 +51,67 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase {
return e;
}
public void testYoutubeExtractAll() throws Exception {
public void testAllInItagPriority() throws Exception {
CrawlURI testUri = createTestUri(TEST_URI);
ArrayList<String> itagPriorityList = new ArrayList<String>();
itagPriorityList.add("44");
itagPriorityList.add("35");
itagPriorityList.add("43");
itagPriorityList.add("34");
itagPriorityList.add("18");
itagPriorityList.add("5");
itagPriorityList.add("36");
itagPriorityList.add("17");
((ExtractorYoutubeFormatStream)extractor).setItagPriority(itagPriorityList);
((ExtractorYoutubeFormatStream)extractor).setExtractLimit(10);
List<String> itagPriorityList = Arrays.asList("44", "35", "43", "34", "18", "5", "36", "17");
extractor().setItagPriority(itagPriorityList);
extractor().setExtractLimit(10);
extractor.process(testUri);
HashSet<Link> expected = new HashSet<Link>();
for (String expectedLinkString : EXPECTED_OUTLINKS_ALL) {
expected.add(new Link(testUri.getUURI(),
UURIFactory.getInstance(expectedLinkString),
HTMLLinkContext.EMBED_MISC, Hop.EMBED)
);
}
Set<Link> expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_ALL);
assertEquals(expected, testUri.getOutLinks());
}
public void testAllNoPriority() throws Exception {
CrawlURI testUri = createTestUri(TEST_URI);
extractor().setExtractLimit(0);
extractor.process(testUri);
Set<Link> expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_ALL);
assertEquals(expected, testUri.getOutLinks());
}
// test that only itags in the priority list are extracted, even though
// extract limit is large
public void testOnlyInItagPriorityBigLimit() throws Exception {
CrawlURI testUri = createTestUri(TEST_URI);
List<String> itagPriorityList = Arrays.asList("44", "35", "43");
extractor().setItagPriority(itagPriorityList);
extractor().setExtractLimit(10);
extractor.process(testUri);
assertEquals(3, testUri.getOutLinks().size());
}
// test that only itags in the priority list are extracted, even though
// extract limit is unset
public void testOnlyInItagPriorityNoLimit() throws Exception {
CrawlURI testUri = createTestUri(TEST_URI);
List<String> itagPriorityList = Arrays.asList("44", "35", "43");
extractor().setItagPriority(itagPriorityList);
extractor().setExtractLimit(0);
extractor.process(testUri);
assertEquals(3, testUri.getOutLinks().size());
}
public void testNoPriorityWithLimit() throws InterruptedException, URIException, UnsupportedEncodingException, IOException {
CrawlURI testUri = createTestUri(TEST_URI);
extractor().setExtractLimit(4);
extractor.process(testUri);
assertEquals(4, testUri.getOutLinks().size());
}
public void testDontExtract() throws URIException, UnsupportedEncodingException, IOException, InterruptedException {
// not a youtube watch url so shouldProcess() will return false
@@ -79,29 +120,48 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase {
assertEquals(Collections.EMPTY_SET, testUri.getOutLinks());
}
public void testYoutubeExtractSubset() throws Exception {
public void testPriority() throws Exception {
CrawlURI testUri = createTestUri(TEST_URI);
ArrayList<String> itagPriorityList = new ArrayList<String>();
itagPriorityList.add("37");
itagPriorityList.add("24");
itagPriorityList.add("34");
itagPriorityList.add("35");
((ExtractorYoutubeFormatStream)extractor).setItagPriority(itagPriorityList);
((ExtractorYoutubeFormatStream)extractor).setExtractLimit(1);
// 37, 24 are not in url_stream_map; 35 appears before 34 in there, but
// with this list we should get 34
extractor().setItagPriority(Arrays.asList("37", "24", "34", "35"));
extractor().setExtractLimit(1);
extractor.process(testUri);
HashSet<Link> expected = new HashSet<Link>();
for (String expectedLinkString : EXPECTED_OUTLINKS_SUBSET) {
expected.add(new Link(testUri.getUURI(),
UURIFactory.getInstance(expectedLinkString),
Set<Link> expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_SUBSET);
assertEquals(expected, testUri.getOutLinks());
}
public void testDefaultItag() throws URIException, UnsupportedEncodingException, IOException, InterruptedException {
CrawlURI testUri = createTestUri(TEST_URI);
extractor().setExtractLimit(1);
assertEquals(Collections.EMPTY_LIST, extractor().getItagPriority());
extractor.process(testUri);
Set<Link> expected = makeLinkSet(testUri, EXPECTED_SINGLE_DEFAULT_OUTLINK);
assertEquals(expected, testUri.getOutLinks());
}
private Set<Link> makeLinkSet(CrawlURI sourceUri, String[] urlStrs) throws URIException {
HashSet<Link> linkSet = new HashSet<Link>();
for (String urlStr : urlStrs) {
linkSet.add(new Link(sourceUri.getUURI(),
UURIFactory.getInstance(urlStr),
HTMLLinkContext.EMBED_MISC, Hop.EMBED)
);
}
assertEquals(expected, testUri.getOutLinks());
return linkSet;
}
}
private ExtractorYoutubeFormatStream extractor() {
return (ExtractorYoutubeFormatStream)extractor;
}
private CrawlURI createTestUri(String urlStr) throws URIException,
UnsupportedEncodingException, IOException {
UURI testUuri = UURIFactory.getInstance(urlStr);
@@ -124,6 +184,7 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase {
return testUri;
}
}
class UnitTestUriLoggerModule implements UriErrorLoggerModule {
final private static Logger LOGGER =
Logger.getLogger(UnitTestUriLoggerModule.class.getName());