From a7dda8b785eec804678a7565edca49a296df6227 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Wed, 5 Jun 2013 16:39:10 -0700 Subject: [PATCH] ExtractorYoutubeFormatStream.java Parse parameters instead of attempting to regex desired values Add priority itag list and extract limit ExtractorYoutubeFormatStreamTest.java Test cases for extracting all videos and extracting single video --- .../ExtractorYoutubeFormatStream.java | 113 +- .../ExtractorYoutubeFormatStreamTest.java | 153 +- .../ExtractorYoutubeFormatStream.txt | 2001 +++++++++++++++++ 3 files changed, 2163 insertions(+), 104 deletions(-) create mode 100644 contrib/src/test/resources/ExtractorYoutubeFormatStream.txt diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java index fb349b91..f84e2af7 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java @@ -5,6 +5,7 @@ import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.util.ArrayList; import java.util.List; +import java.util.LinkedHashMap; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; @@ -72,14 +73,13 @@ public class ExtractorYoutubeFormatStream extends ContentExtractor { @Override protected boolean shouldExtract(CrawlURI uri){ - return true; - // if (uri.getFetchStatus() != 200) { - // //verify we are a youtube watch page - // return false; - // } - // else { - // return true; - // } + if (uri.getFetchStatus() != 200) { + //TODO: verify we are a youtube watch page + return false; + } + else { + return true; + } } @Override protected boolean innerExtract(CrawlURI uri) { @@ -96,7 +96,7 @@ public class ExtractorYoutubeFormatStream extends ContentExtractor { if (matcher.find()) { String jsonStr = matcher.group(1); - logger.info("Just Extracted: "+jsonStr); + //logger.fine("Just Extracted: "+jsonStr); JSONObject json; try { json = new JSONObject(jsonStr); @@ -105,54 +105,65 @@ public class ExtractorYoutubeFormatStream extends ContentExtractor { if(args.has("url_encoded_fmt_stream_map")) { String stream_map = args.getString("url_encoded_fmt_stream_map"); - logger.info("Just Extracted: "+stream_map); + //logger.info("Just Extracted: "+stream_map); - String[] videos = stream_map.split(","); + String[] rawVideoList = stream_map.split(","); + LinkedHashMap parsedVideoMap = new LinkedHashMap(); - for(int i=0; i < videos.length; i++) { - String[] videoParams = videos[i].split("\u0026"); + //Parse Video Map into itag,url pair + for(int i=0; i < rawVideoList.length; i++) { + String[] videoParams = rawVideoList[i].split("\\u0026"); + String videoURLParam, itagParam, sigParam; + videoURLParam = itagParam = sigParam = ""; - Matcher videoMatchPre = TextUtils.getMatcher("(?is)sig=([^&]*).*url=(http[^&]*)", videos[i]); - Matcher videoMatchPost = TextUtils.getMatcher("(?is)url=(http[^&]*).*sig=([^&]*)", videos[i]); - String videoUri=""; - String prefix,suffix; - - if(videoMatchPre.find()) { - prefix = videoMatchPre.group(2); - suffix = videoMatchPre.group(1); - } - else if(videoMatchPost.find()) { - prefix = videoMatchPost.group(1); - suffix = videoMatchPost.group(2); - } - else - return false; - - try { - videoUri = URLDecoder.decode(prefix+"%26signature="+suffix, "UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); // impossible + for(String param : videoParams){ + + String[] keyValuePair = param.split("="); + if(keyValuePair.length!=2){ + logger.warning("Invalid Video Parameter: "+param); + continue; + } + + if(keyValuePair[0].equals("url")) + videoURLParam=keyValuePair[1]; + if(keyValuePair[0].equals("itag")) + itagParam=keyValuePair[1]; + if(keyValuePair[0].equals("sig")) + sigParam=keyValuePair[1]; } - TextUtils.recycleMatcher(videoMatchPre); - TextUtils.recycleMatcher(videoMatchPost); + if(videoURLParam.length()>0 && itagParam.length()>0 && sigParam.length()>0) { + try { + String fixupURL = URLDecoder.decode(videoURLParam+"%26signature="+sigParam, "UTF-8"); + parsedVideoMap.put(itagParam,fixupURL); + } + catch(java.io.UnsupportedEncodingException e) { + logger.warning("Error decoding youtube video URL: "+videoURLParam+"%26signature="+sigParam); + } + } + } + // for(String itag : getItagPriority()){ + // logger.warning("itag Priority List has: "+itag); + // } + // for(String itag : parsedVideoMap.keySet()) { + // logger.warning("parsed List has itag: "+itag); + // } + //Add videos as outlinks by priority list + int extractionCount=0; + for(String itag : getItagPriority()) { + if(parsedVideoMap.containsKey(itag) && extractionCountvar ytplayer = ytplayer || {};ytplayer.config = {\"params\": {\"bgcolor\": \"#000000\", \"allowfullscreen\": \"true\", \"allowscriptaccess\": \"always\"}, \"assets\": {\"js\": \"http:\\/\\/s.ytimg.com\\/yts\\/jsbin\\", -"/html5player-vflEMhIP1.js\", \"css\": \"http:\\/\\/s.ytimg.com\\/yts\\/cssbin\\/www-player-vflqpZ-ak.css\", \"html\": \"\\/html5_player_template\"}, \"attrs\": {\"id\": \"movie_player\"}, \"url_v8\": \"http:\\/\\/s.ytimg.com\\/yts\\", -"/swfbin\\/cps-vflqv0phk.swf\", \"url_v9as2\": \"http:\\/\\/s.ytimg.com\\/yts\\/swfbin\\/cps-vflqv0phk.swf\", \"sts\": 1585, \"url\": \"http:\\/\\/s.ytimg.com\\/yts\\/swfbin\\/watch_as3-vfl2gg8hE.swf\", \"args\": {\"enablejsapi\": ", -"1, \"csi_page_type\": \"watch,watch7\", \"timestamp\": 1370455277, \"t\": \"vjVQa1PpcFNDhHLAnDKnZQTDaXu4uUWaFAfST80nc_o=\", \"c\": \"WEB\", \"rvs\": \"title=Big+Think+Interview+With+George+Lois\\u0026view_count=2%2C428\\u00", -"26featured=1\\u0026feature_type=fvwp\\u0026author=bigthink\\u0026id=-92PkAXJIqQ\\u0026length_seconds=3183,length_seconds=3873\\u0026title=Bill%27s+Design+Talks%3A+Nonobject-+A+Radical+New+Approach+to+Design\\u0", -"026author=cooperhewitt\\u0026id=iQzYRC9fGE8\\u0026view_count=1%2C376,length_seconds=2878\\u0026title=Bill%27s+Design+Talks%3A+Pentagram\\u0026author=cooperhewitt\\u0026id=Jtqi0F3cDaU\\u0026view_count=8%2C721,le", -"ngth_seconds=4183\\u0026title=Bill%27s+Design+Talks%3A+Emily+Oberman\\u0026author=cooperhewitt\\u0026id=WwA1qGubdHU\\u0026view_count=848,length_seconds=4143\\u0026title=Bill%27s+Design+Talks%3A+Robert+Wong%2C+", -"Google+CreativeLab\\u0026author=cooperhewitt\\u0026id=6vIPk6vIwv0\\u0026view_count=3%2C987,length_seconds=5260\\u0026title=Designing+Logos+-+Graphic+Design+for+Everyone\\u0026author=creativelive\\u0026id=d_mDEv", -"8fMug\\u0026view_count=4%2C454,length_seconds=3225\\u0026title=History+of+Industrial+Design+Lecture+%235%3A+Efficiency\\u0026author=HistoryofID\\u0026id=z8MhAm-FwX4\\u0026view_count=1%2C881,length_seconds=5318", -"\\u0026title=Academy+of+Art+University+%7C+School+of+Graphic+Design+%7C+Hybrid+Design%3A+Work+%26+Play+Panel\\u0026author=academyofartu\\u0026id=ErrmZ_xQ5hs\\u0026view_count=1%2C998,length_seconds=873\\u0026ti", -"tle=Designing+Graphic+Design+History\\u0026author=RockPaperInk\\u0026id=vkFkb1ghwWo\\u0026view_count=5%2C878,length_seconds=3847\\u0026title=3.+Mobile+User+Experience+Design\\u0026author=StanfordUniversity\\u00", -"26id=SEwCRpTEgA0\\u0026view_count=29%2C591,length_seconds=965\\u0026title=Graphic+Design+Tutorial%3A+Typography+and+Design\\u0026author=UltramanToronto\\u0026id=mBoVoj5jLfc\\u0026view_count=4%2C816,length_seco", -"nds=4232\\u0026title=Bill%27s+Design+Talks%3A+Annabelle+Selldorf\\u0026author=cooperhewitt\\u0026id=87nH6T01Zz4\\u0026view_count=635\", \"tmi\": \"1\", \"storyboard_spec\": \"http:\\/\\/i4.ytimg.com\\/sb\\/_BFJN62hZp0\\/s", -"toryboard3_L$L\\/$N.jpg|48#27#100#10#10#0#default#-gk1SKNrHE-Tmy9iS7nuPTvFQuA|67#45#543#10#10#10000#M$M#xhaAoys0tYIQZ5-72FPCgETNlio|135#90#543#5#5#10000#M$M#86YKPK4UMPTNtqZalcHO0ika8yA\", \"abd\": \"1\", \"title", -"\": \"Bill's Design Talks: Graphic Design\u2014 Now in Production\", \"cc3_module\": \"http:\\/\\/s.ytimg.com\\/yts\\/swfbin\\/subtitles3_module-vfl8mhMFB.swf\", \"autohide\": \"2\", \"sendtmp\": \"1\", \"referrer\": null, \"pl", -"id\": \"AATea_wEpIEQtO3r\", \"ttsurl\": \"http:\\/\\/www.youtube.com\\/api\\/timedtext?hl=en_US\\u0026signature=C5F6882E3BB97A13088D7C6134C008EDE1C239F4.C53C55934FC98E4D8A9D40CCD6B86819F94A66CD\\u0026expire=1370480477\\u0026caps=asr\\u0026key=yttt1\\u0026v=_BFJN62hZp0\\u0026sparams=asr_langs%2Ccaps%2Cv%2Cexpire\\u0026asr_langs=en%2Cko%2Cja%2Cde%2Cpt%2Cnl%2Cfr%2Cru%2Ces%2Cit\", \"cc_font\": \"Arial Unicode MS, arial, verdana, _sans\", \"sk\": \"Pn-rDSo7hZBCDhj9vVw2lp7EGCA9DWQ5C\", \"watermark\": \",http:\\/\\/s.ytimg.com\\/yts\\/img\\/watermark\\/youtube_watermark-vflHX6b6E.png,http:\\/\\/s.ytimg.com\\/yts\\/img\\/watermark\\/youtube_hd_watermark-vflAzLcD6.png\", \"no_get_video_log\": \"1\", \"hl\": \"en_US\", \"ptk\": \"youtube_none\", \"fexp\": \"911418,932402,923500,901472,900352,924605,928201,901208,929123,929915,929906,925714,929919,929119,931202,932802,928017,912512,912518,911416,906906,904476,930807,919373,906836,933701,926403,912711,929606,910075\", \"fmt_list\": \"44\\/854x480\\/99\\/0\\/0,35\\/854x480\\/9\\/0\\/115,43\\/640x360\\/99\\/0\\/0,34\\/640x360\\/9\\/0\\/115,18\\/640x360\\/9\\/0\\/115,5\\/320x240\\/7\\/0\\/0,36\\/320x240\\/99\\/0\\/0,17\\/176x144\\/99\\/0\\/0\", \"showpopout\": 1, \"video_id\": \"_BFJN62hZp0\", \"playlist_module\": \"http:\\/\\/s.ytimg.com\\/yts\\/swfbin\\/playlist_module-vflQiukBo.swf\", \"cc_asr\": 1, \"keywords\": \"graphic design,contemporary,exhibition,software,diy,tech,governors island,blauvelt,moggridge,lupton,cooper-hewitt,smithsonian,posterwall,lust,nieuwenhuizen,talk,panel,bill's design talks,greene space,nyc\", \"ucid\": \"UCBCaO28GYGr7IoKVjT4Jx5A\", \"cc_module\": \"http:\\/\\/s.ytimg.com\\/yts\\/swfbin\\/subtitle_module-vfldzX-hd.swf\", \"account_playback_token\": \"joYXG92N8rVztPTD8W-zEszJCQx8MTM3MDU0MTY3N0AxMzcwNDU1Mjc3\", \"endscreen_module\": \"http:\\/\\/s.ytimg.com\\/yts\\/swfbin\\/endscreen-vflKOX8EP.swf\", \"enablecsi\": \"1\", \"pltype\": \"contentugc\", \"tk\": \"1\", \"share_icons\": \"http:\\/\\/s.ytimg.com\\/yts\\/swfbin\\/sharing-vflF4tO1T.swf\", \"vq\": \"auto\", \"allow_embed\": 1, \"is_html5_mobile_device\": false, \"length_seconds\": 5420, \"url_encoded_fmt_stream_map\": \"fallback_host=tc.v23.cache7.c.youtube.com\\u0026sig=097BABD561F197C107AE9BE57C3CE4F2A9193C57.25249074A61F72F639C4635C31DA48E2D24F125F\\u0026itag=44\\u0026url=http%3A%2F%2Fr3---sn-a5m7znek.c.youtube.com%2Fvideoplayback%3Fkey%3Dyt1%26ip%3D208.70.31.237%26sver%3D3%26mv%3Dm%26itag%3D44%26source%3Dyoutube%26fexp%3D911418%252C932402%252C923500%252C901472%252C900352%252C924605%252C928201%252C901208%252C929123%252C929915%252C929906%252C925714%252C929919%252C929119%252C931202%252C932802%252C928017%252C912512%252C912518%252C911416%252C906906%252C904476%252C930807%252C919373%252C906836%252C933701%252C926403%252C912711%252C929606%252C910075%26expire%3D1370478870%26newshard%3Dyes%26ms%3Dau%26upn%3Dko-PmxWJzEg%26ratebypass%3Dyes%26id%3Dfc114937ada1669d%26mt%3D1370455230%26sparams%3Dcp%252Cid%252Cip%252Cipbits%252Citag%252Cratebypass%252Csource%252Cupn%252Cexpire%26ipbits%3D8%26cp%3DU0hWRVRSVl9MSkNONl9MTFpJOm8tNWxnVEptdFY2\\u0026quality=large\\u0026type=video%2Fwebm%3B+codecs%3D%22vp8.0%2C+vorbis%22,fallback_host=tc.v2.cache1.c.youtube.com\\u0026sig=745286D6E79C965AE1AFC046A2348778F3DBD7E1.35BF2F520B7A42CADF2F076C72F3943C38CB85A8\\u0026itag=35\\u0026url=http%3A%2F%2Fr3---sn-a5m7znek.c.youtube.com%2Fvideoplayback%3Falgorithm%3Dthrottle-factor%26ip%3D208.70.31.237%26sver%3D3%26factor%3D1.25%26mt%3D1370455230%26itag%3D35%26source%3Dyoutube%26fexp%3D911418%252C932402%252C923500%252C901472%252C900352%252C924605%252C928201%252C901208%252C929123%252C929915%252C929906%252C925714%252C929919%252C929119%252C931202%252C932802%252C928017%252C912512%252C912518%252C911416%252C906906%252C904476%252C930807%252C919373%252C906836%252C933701%252C926403%252C912711%252C929606%252C910075%26expire%3D1370478870%26newshard%3Dyes%26ms%3Dau%26upn%3Dko-PmxWJzEg%26id%3Dfc114937ada1669d%26burst%3D40%26sparams%3Dalgorithm%252Cburst%252Ccp%252Cfactor%252Cid%252Cip%252Cipbits%252Citag%252Csource%252Cupn%252Cexpire%26mv%3Dm%26ipbits%3D8%26key%3Dyt1%26cp%3DU0hWRVRSVl9MSkNONl9MTFpJOm8tNWxnVEptdFY2\\u0026quality=large\\u0026type=video%2Fx-flv,fallback_host=tc.v22.cache3.c.youtube.com\\u0026sig=AC8D17DAF1D2553C15F4E23D24D1C7013A0637FB.596D6011AB9AA4E6025F7784E204BD6BD2FC9408\\u0026itag=43\\u0026url=http%3A%2F%2Fr3---sn-a5m7znek.c.youtube.com%2Fvideoplayback%3Fkey%3Dyt1%26ip%3D208.70.31.237%26sver%3D3%26mv%3Dm%26itag%3D43%26source%3Dyoutube%26fexp%3D911418%252C932402%252C923500%252C901472%252C900352%252C924605%252C928201%252C901208%252C929123%252C929915%252C929906%252C925714%252C929919%252C929119%252C931202%252C932802%252C928017%252C912512%252C912518%252C911416%252C906906%252C904476%252C930807%252C919373%252C906836%252C933701%252C926403%252C912711%252C929606%252C910075%26expire%3D1370478870%26newshard%3Dyes%26ms%3Dau%26upn%3Dko-PmxWJzEg%26ratebypass%3Dyes%26id%3Dfc114937ada1669d%26mt%3D1370455230%26sparams%3Dcp%252Cid%252Cip%252Cipbits%252Citag%252Cratebypass%252Csource%252Cupn%252Cexpire%26ipbits%3D8%26cp%3DU0hWRVRSVl9MSkNONl9MTFpJOm8tNWxnVEptdFY2\\u0026quality=medium\\u0026type=video%2Fwebm%3B+codecs%3D%22vp8.0%2C+vorbis%22,fallback_host=tc.v11.cache4.c.youtube.com\\u0026sig=4A6FEC2DB96926893C2359E8ED680A1E5B21B71C.74B3D1DC45C2F1DA3CAFB4218704AD2A4B61D4C2\\u0026itag=34\\u0026url=http%3A%2F%2Fr3---sn-a5m7znek.c.youtube.com%2Fvideoplayback%3Falgorithm%3Dthrottle-factor%26ip%3D208.70.31.237%26sver%3D3%26factor%3D1.25%26mt%3D1370455230%26itag%3D34%26source%3Dyoutube%26fexp%3D911418%252C932402%252C923500%252C901472%252C900352%252C924605%252C928201%252C901208%252C929123%252C929915%252C929906%252C925714%252C929919%252C929119%252C931202%252C932802%252C928017%252C912512%252C912518%252C911416%252C906906%252C904476%252C930807%252C919373%252C906836%252C933701%252C926403%252C912711%252C929606%252C910075%26expire%3D1370478870%26newshard%3Dyes%26ms%3Dau%26upn%3Dko-PmxWJzEg%26id%3Dfc114937ada1669d%26burst%3D40%26sparams%3Dalgorithm%252Cburst%252Ccp%252Cfactor%252Cid%252Cip%252Cipbits%252Citag%252Csource%252Cupn%252Cexpire%26mv%3Dm%26ipbits%3D8%26key%3Dyt1%26cp%3DU0hWRVRSVl9MSkNONl9MTFpJOm8tNWxnVEptdFY2\\u0026quality=medium\\u0026type=video%2Fx-flv,fallback_host=tc.v17.cache2.c.youtube.com\\u0026sig=D2B301B46D87D1980C65FF9FEF67BD77CAC05F0F.5C76D533CC6B3CB06E114E9A97D4F62710EFC102\\u0026itag=18\\u0026url=http%3A%2F%2Fr3---sn-a5m7znek.c.youtube.com%2Fvideoplayback%3Fkey%3Dyt1%26ip%3D208.70.31.237%26sver%3D3%26mv%3Dm%26itag%3D18%26source%3Dyoutube%26fexp%3D911418%252C932402%252C923500%252C901472%252C900352%252C924605%252C928201%252C901208%252C929123%252C929915%252C929906%252C925714%252C929919%252C929119%252C931202%252C932802%252C928017%252C912512%252C912518%252C911416%252C906906%252C904476%252C930807%252C919373%252C906836%252C933701%252C926403%252C912711%252C929606%252C910075%26expire%3D1370478870%26newshard%3Dyes%26ms%3Dau%26upn%3Dko-PmxWJzEg%26ratebypass%3Dyes%26id%3Dfc114937ada1669d%26mt%3D1370455230%26sparams%3Dcp%252Cid%252Cip%252Cipbits%252Citag%252Cratebypass%252Csource%252Cupn%252Cexpire%26ipbits%3D8%26cp%3DU0hWRVRSVl9MSkNONl9MTFpJOm8tNWxnVEptdFY2\\u0026quality=medium\\u0026type=video%2Fmp4%3B+codecs%3D%22avc1.42001E%2C+mp4a.40.2%22,fallback_host=tc.v19.cache3.c.youtube.com\\u0026sig=6BBDDD158B7FDA10F5ECE1CBB5F784AF7BB341E0.4387402752DF53E2D1C5185503F6C0D96A4BBB5F\\u0026itag=5\\u0026url=http%3A%2F%2Fr3---sn-a5m7znek.c.youtube.com%2Fvideoplayback%3Falgorithm%3Dthrottle-factor%26ip%3D208.70.31.237%26sver%3D3%26factor%3D1.25%26mt%3D1370455230%26itag%3D5%26source%3Dyoutube%26fexp%3D911418%252C932402%252C923500%252C901472%252C900352%252C924605%252C928201%252C901208%252C929123%252C929915%252C929906%252C925714%252C929919%252C929119%252C931202%252C932802%252C928017%252C912512%252C912518%252C911416%252C906906%252C904476%252C930807%252C919373%252C906836%252C933701%252C926403%252C912711%252C929606%252C910075%26expire%3D1370478870%26newshard%3Dyes%26ms%3Dau%26upn%3Dko-PmxWJzEg%26id%3Dfc114937ada1669d%26burst%3D40%26sparams%3Dalgorithm%252Cburst%252Ccp%252Cfactor%252Cid%252Cip%252Cipbits%252Citag%252Csource%252Cupn%252Cexpire%26mv%3Dm%26ipbits%3D8%26key%3Dyt1%26cp%3DU0hWRVRSVl9MSkNONl9MTFpJOm8tNWxnVEptdFY2\\u0026quality=small\\u0026type=video%2Fx-flv,fallback_host=tc.v19.cache4.c.youtube.com\\u0026sig=7C7DF993A78D3CA3D1CB80340E8EB52EFE9084A5.12F708A4B708EC0290B697F54079B31CF067B83C\\u0026itag=36\\u0026url=http%3A%2F%2Fr3---sn-a5m7znek.c.youtube.com%2Fvideoplayback%3Falgorithm%3Dthrottle-factor%26ip%3D208.70.31.237%26sver%3D3%26factor%3D1.25%26mt%3D1370455230%26itag%3D36%26source%3Dyoutube%26fexp%3D911418%252C932402%252C923500%252C901472%252C900352%252C924605%252C928201%252C901208%252C929123%252C929915%252C929906%252C925714%252C929919%252C929119%252C931202%252C932802%252C928017%252C912512%252C912518%252C911416%252C906906%252C904476%252C930807%252C919373%252C906836%252C933701%252C926403%252C912711%252C929606%252C910075%26expire%3D1370478870%26newshard%3Dyes%26ms%3Dau%26upn%3Dko-PmxWJzEg%26id%3Dfc114937ada1669d%26burst%3D40%26sparams%3Dalgorithm%252Cburst%252Ccp%252Cfactor%252Cid%252Cip%252Cipbits%252Citag%252Csource%252Cupn%252Cexpire%26mv%3Dm%26ipbits%3D8%26key%3Dyt1%26cp%3DU0hWRVRSVl9MSkNONl9MTFpJOm8tNWxnVEptdFY2\\u0026quality=small\\u0026type=video%2F3gpp%3B+codecs%3D%22mp4v.20.3%2C+mp4a.40.2%22,fallback_host=tc.v3.cache3.c.youtube.com\\u0026sig=787424AE0AA71635806342CDEDF148D340601AF5.329DEB777CDF474FEA3B4D4A041F5188FA892981\\u0026itag=17\\u0026url=http%3A%2F%2Fr3---sn-a5m7znek.c.youtube.com%2Fvideoplayback%3Falgorithm%3Dthrottle-factor%26ip%3D208.70.31.237%26sver%3D3%26factor%3D1.25%26mt%3D1370455230%26itag%3D17%26source%3Dyoutube%26fexp%3D911418%252C932402%252C923500%252C901472%252C900352%252C924605%252C928201%252C901208%252C929123%252C929915%252C929906%252C925714%252C929919%252C929119%252C931202%252C932802%252C928017%252C912512%252C912518%252C911416%252C906906%252C904476%252C930807%252C919373%252C906836%252C933701%252C926403%252C912711%252C929606%252C910075%26expire%3D1370478870%26newshard%3Dyes%26ms%3Dau%26upn%3Dko-PmxWJzEg%26id%3Dfc114937ada1669d%26burst%3D40%26sparams%3Dalgorithm%252Cburst%252Ccp%252Cfactor%252Cid%252Cip%252Cipbits%252Citag%252Csource%252Cupn%252Cexpire%26mv%3Dm%26ipbits%3D8%26key%3Dyt1%26cp%3DU0hWRVRSVl9MSkNONl9MTFpJOm8tNWxnVEptdFY2\\u0026quality=small\\u0026type=video%2F3gpp%3B+codecs%3D%22mp4v.20.3%2C+mp4a.40.2%22\", \"cr\": \"US\"}, \"min_version\": \"8.0.0\", \"html5\": false};" - - }; + protected static final String[] EXPECTED_OUTLINKS_SUBSET = { + "http://r3---sn-a5m7znek.c.youtube.com/generate_204", + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=34&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=78D14935180C9DA87E1C562719525D0BB6BE21F9.9BC13425338E5DD6C255EA77136CC424830BCD21" + + }; @Override protected Extractor makeExtractor() { ExtractorYoutubeFormatStream e = new ExtractorYoutubeFormatStream(); UriErrorLoggerModule ulm = new UnitTestUriLoggerModule(); e.setLoggerModule(ulm); + + return e; + } + public void testYoutubeExtractAll() throws Exception { + UURI testUuri = UURIFactory.getInstance(TEST_URI); + CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC); + + InputStream is = ExtractorYoutubeFormatStreamTest.class.getClassLoader().getResourceAsStream("ExtractorYoutubeFormatStream.txt"); + BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8")); + StringBuilder content = new StringBuilder(); + String line = ""; + while ((line = reader.readLine()) != null) { + content.append(line); + } + Recorder recorder = createRecorder(content.toString(), "UTF-8"); + IOUtils.closeQuietly(is); + + testUri.setContentType("text/html"); + testUri.setFetchStatus(200); + testUri.setRecorder(recorder); + testUri.setContentSize(content.length()); + + ArrayList itagPriorityList = new ArrayList(); + itagPriorityList.add("44"); + itagPriorityList.add("35"); + itagPriorityList.add("43"); + itagPriorityList.add("34"); + itagPriorityList.add("18"); + itagPriorityList.add("5"); + itagPriorityList.add("36"); + itagPriorityList.add("17"); + ((ExtractorYoutubeFormatStream)extractor).setItagPriority(itagPriorityList); + ((ExtractorYoutubeFormatStream)extractor).setExtractLimit(10); + + extractor.process(testUri); + + HashSet expected = new HashSet(); + for (String expectedLinkString : EXPECTED_OUTLINKS_ALL) { + expected.add(new Link(testUri.getUURI(), + UURIFactory.getInstance(expectedLinkString), + HTMLLinkContext.EMBED_MISC, Hop.EMBED) + ); + } + assertEquals(expected, testUri.getOutLinks()); + } + + public void testYoutubeExtractSubset() throws Exception { + UURI testUuri = UURIFactory.getInstance(TEST_URI); + CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC); + + InputStream is = ExtractorYoutubeFormatStreamTest.class.getClassLoader().getResourceAsStream("ExtractorYoutubeFormatStream.txt"); + BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8")); + StringBuilder content = new StringBuilder(); + String line = ""; + while ((line = reader.readLine()) != null) { + content.append(line); + } + Recorder recorder = createRecorder(content.toString(), "UTF-8"); + IOUtils.closeQuietly(is); + + testUri.setContentType("text/html"); + testUri.setFetchStatus(200); + testUri.setRecorder(recorder); + testUri.setContentSize(content.length()); ArrayList itagPriorityList = new ArrayList(); itagPriorityList.add("37"); itagPriorityList.add("24"); - itagPriorityList.add("35"); itagPriorityList.add("34"); - - e.setItagPriority(itagPriorityList); - e.setExtractLimit(8); + itagPriorityList.add("35"); + ((ExtractorYoutubeFormatStream)extractor).setItagPriority(itagPriorityList); + ((ExtractorYoutubeFormatStream)extractor).setExtractLimit(1); - return e; - } - public void testYoutubeExtractExample() throws Exception { - UURI testUuri = UURIFactory.getInstance(TEST_URI); - CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC); - StringBuilder content = new StringBuilder(); - for (String chunk: TEST_CONTENT_CHUNKS) { - content.append(chunk); - } - Recorder recorder = createRecorder(content.toString(), "UTF-8"); - testUri.setContentType("text/html"); - //set to 200 - testUri.setRecorder(recorder); - testUri.setContentSize(content.length()); - extractor.process(testUri); - for (String expectedLinkString: EXPECTED_OUTLINKS) { - Link expectedLink = new Link(testUri.getUURI(), + HashSet expected = new HashSet(); + for (String expectedLinkString : EXPECTED_OUTLINKS_SUBSET) { + expected.add(new Link(testUri.getUURI(), UURIFactory.getInstance(expectedLinkString), - HTMLLinkContext.EMBED_MISC, Hop.EMBED); - assertTrue(testUri.getOutLinks().contains(expectedLink)); + HTMLLinkContext.EMBED_MISC, Hop.EMBED) + ); } + assertEquals(expected, testUri.getOutLinks()); + } } class UnitTestUriLoggerModule implements UriErrorLoggerModule { @@ -98,6 +140,11 @@ class UnitTestUriLoggerModule implements UriErrorLoggerModule { LOGGER.log(Level.INFO, info); System.out.println("INFO - "+info); } + public void fine(String info) { + System.out.println("Fine - "+info); + LOGGER.log(Level.FINE, info); + + } public void logUriError(URIException e, UURI u, CharSequence l) { LOGGER.log(Level.INFO, u.toString(), e); System.out.println("Info - "+ u.toString()+" "+e.toString()); diff --git a/contrib/src/test/resources/ExtractorYoutubeFormatStream.txt b/contrib/src/test/resources/ExtractorYoutubeFormatStream.txt new file mode 100644 index 00000000..72119847 --- /dev/null +++ b/contrib/src/test/resources/ExtractorYoutubeFormatStream.txt @@ -0,0 +1,2001 @@ + + + + Bill's Design Talks: Graphic Design— Now in Production - YouTube + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
Upload
+ +
+ + + +
+ + + + +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+
+ +
+

+ This video is unavailable. +

+
+
+
+ + +
+ + + +
+ + + + + + + +
+ +
+ +
+ +
+
+
+
+
+

+ + + + + + + + Bill's Design Talks: Graphic Design— Now in Production + + +

+
+ +
+ + + + cooperhewitt + + + + + +cooperhewitt·427 videos
2,688 +
+ 1,200 + + +
+
+
+
+ + Like + +     Dislike + 0 + + + +
+
+
+
+

Sign in to YouTube

+
+ Sign in with your Google Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to like cooperhewitt's video. + +
+
    +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
+ +
+
+

Sign in to YouTube

+
+ Sign in with your Google Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to dislike cooperhewitt's video. + +
+
    +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
+ +
+ +
+
+ + + + + + + + + + +
+

Sign in to YouTube

+
+ Sign in with your Google Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to add cooperhewitt's video to your playlist. + +
+
    +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
  • +
    +
    • +
+ +
+ + + + + + + + + + + + + + + + +
+
+ +
+
+
+
+
+

+ Published on Jun 28, 2012 + +

+
+

Find out how designers today are rethinking the aesthetics, process, and public of graphic design by creating their own software and systems. Join Cooper-Hewitt Director Bill Moggridge, Walker Art Center Curator Andrew Blauvelt and interaction designer Dimitri Nieuwenhuizen in a lively discussion. Dimitri is a member of Lust, the innovative Dutch design collective that created the large-scale interactive project Posterwall for the Twenty-first Century, on view in Cooper-Hewitt's new exhibition Graphic Design—Now In Production, on view every weekend through Labor Day on Governors Island, New York City. The exhibition was co-organized by Cooper-Hewitt and Walker Art Center.

+
+
+
    +
  • +

    +Category +

    +
    +

    Education

    + +
    +
    • + + +
  • +

    License

    +
    +

    +Standard YouTube License +

    + +
    • + +
+
+ +
+
    + +
+
+ +
+
+ +
+
+ +
+
+
+
+ +
+
+
+ + +
+
+
+ +
+ +
+
+ + +
+
+ +
+
+
+ + +
+
+
+
+ +
+ + + + +
+
+The interactive transcript could not be loaded. +
+ + + +
+
+ +
+ +
+
+ + +
+
+ +
+
+ + +
+
+ +
+ +
+ +
+
+ Ratings have been disabled for this video. +
+ +
+ +
+
+ Rating is available when the video has been rented. +
+ +
+ +
+
+ This feature is not available right now. Please try again later. +
+
+ + +
+ +
+
+
+ + +
+ +
+

+ + All Comments (1) + + +

+ + +
+ Sign in now to post a comment! + +
+ + +
    + + + +
  • + + + + + + GraphicDesignStuff + + + + + + + + + +
    +

    + + GraphicDesignStuff + + + + 2 weeks ago + + +

    + + +
    +

    Excellent video!

    +

    Had to shared on our Graphic Design Page on FaceBook.

    +

    facebook(dot)com/GraphicDesign­Stuff

    +

    

    + +
    + +
    + + · + + +
    +

    Sign in to YouTube

    +
    + Sign in with your YouTube Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to rate GraphicDesignStuff's comment. + +
    +
      +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    + +
    +
    +

    Sign in to YouTube

    +
    + Sign in with your YouTube Account (YouTube, Google+, Gmail, Orkut, Picasa, or Chrome) to rate GraphicDesignStuff's comment. + +
    +
      +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    • +
      +
      • +
    + +
    +     +
    + +
    + + +
    • + +
+ +
+ + + + +
    +
  • Loading comment...
    • +
+
+
Loading...
+ + + +
+ + + +
+
+ + + + +
+ +
+ +
+ + +
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+
Loading...
+
+ +
+
+
+
+ +
+
+
+
+
+
+
Working...
+
+
+ +
+
+
+
+
+ + + +
+ to add this to Watch Later + +
+ + + + + + + + + +
+