diff --git a/contrib/pom.xml b/contrib/pom.xml
index 5a1e1ba6..fa1877c6 100644
--- a/contrib/pom.xml
+++ b/contrib/pom.xml
@@ -66,11 +66,6 @@
rethinkdb-driver
2.3.3
-
- com.googlecode.json-simple
- json-simple
- 1.1.1
-
diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java
index 341a8ce6..0fc0d070 100644
--- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java
+++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java
@@ -25,7 +25,6 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
-import java.io.StringWriter;
import java.net.URI;
import java.util.Arrays;
import java.util.concurrent.Callable;
@@ -48,16 +47,12 @@ import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.util.ArchiveUtils;
import org.archive.util.MimetypeUtils;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.Lifecycle;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonStreamParser;
-import com.google.gson.internal.Streams;
-import com.google.gson.stream.JsonWriter;
-
/**
* Extracts links to media by running youtube-dl in a subprocess. Runs only on
* html.
@@ -167,30 +162,28 @@ public class ExtractorYoutubeDL extends Extractor
logCapturedVideo(uri, ydlAnnotation);
}
} else {
- JsonObject ydlJson = runYoutubeDL(uri);
+ JSONObject ydlJson = runYoutubeDL(uri);
if (ydlJson != null && (ydlJson.has("entries") || ydlJson.has("url"))) {
- Iterable jsonEntries;
+ JSONArray jsonEntries;
if (ydlJson.has("entries")) {
- jsonEntries = ydlJson.getAsJsonArray("entries");
+ jsonEntries = ydlJson.getJSONArray("entries");
} else {
- jsonEntries = Arrays.asList(ydlJson);
+ jsonEntries = new JSONArray(Arrays.asList(ydlJson));
}
- int count = 0;
- for (JsonElement jsonE: jsonEntries) {
- count += 1;
- JsonObject jsonO = (JsonObject) jsonE;
+ for (int i = 0; i < jsonEntries.length(); i++) {
+ JSONObject jsonO = (JSONObject) jsonEntries.get(i);
// media url
if (jsonO.get("url") != null) {
- String videoUrl = jsonO.get("url").getAsString();
+ String videoUrl = jsonO.getString("url");
addVideoOutlink(uri, jsonO, videoUrl);
}
// make sure we extract watch page links from youtube playlists,
// and equivalent for other sites
if (jsonO.get("webpage_url") != null) {
- String webpageUrl = jsonO.get("webpage_url").getAsString();
+ String webpageUrl = jsonO.getString("webpage_url");
try {
UURI dest = UURIFactory.getInstance(uri.getUURI(), webpageUrl);
CrawlURI link = uri.createCrawlURI(dest, LinkContext.NAVLINK_MISC,
@@ -205,14 +198,14 @@ public class ExtractorYoutubeDL extends Extractor
// XXX this can be large, consider using a RecordingOutputStream
uri.getData().put("ydlJson", ydlJson);
- String annotation = "youtube-dl:" + count;
+ String annotation = "youtube-dl:" + jsonEntries.length();
uri.getAnnotations().add(annotation);
logContainingPage(uri, annotation);
}
}
}
- protected void addVideoOutlink(CrawlURI uri, JsonObject json,
+ protected void addVideoOutlink(CrawlURI uri, JSONObject jsonO,
String videoUrl) {
try {
UURI dest = UURIFactory.getInstance(uri.getUURI(), videoUrl);
@@ -221,9 +214,9 @@ public class ExtractorYoutubeDL extends Extractor
// annotation
String annotation = "youtube-dl:1/1";
- if (!json.get("playlist_index").isJsonNull()) {
- annotation = "youtube-dl:" + json.get("playlist_index") + "/"
- + json.get("n_entries");
+ if (jsonO.opt("playlist_index") != null) {
+ annotation = "youtube-dl:" + jsonO.get("playlist_index") + "/"
+ + jsonO.get("n_entries");
}
link.getAnnotations().add(annotation);
@@ -350,7 +343,7 @@ public class ExtractorYoutubeDL extends Extractor
return output;
}
- protected JsonObject runYoutubeDL(CrawlURI uri) {
+ protected JSONObject runYoutubeDL(CrawlURI uri) {
/*
* --format=best
*
@@ -399,13 +392,10 @@ public class ExtractorYoutubeDL extends Extractor
proc.destroyForcibly();
}
- JsonStreamParser parser = new JsonStreamParser(output.stdout);
- JsonObject ydlJson = null;
try {
- if (parser.hasNext()) {
- ydlJson = (JsonObject) parser.next();
- }
- } catch (JsonParseException e) {
+ JSONObject ydlJson = new JSONObject(output.stdout);
+ return ydlJson;
+ } catch (JSONException e) {
// sometimes we get no output at all from youtube-dl, which
// manifests as a JsonIOException
logger.log(Level.FINE,
@@ -415,8 +405,6 @@ public class ExtractorYoutubeDL extends Extractor
e);
return null;
}
-
- return ydlJson;
}
@Override
@@ -480,13 +468,10 @@ public class ExtractorYoutubeDL extends Extractor
recordInfo.setMimetype("application/vnd.youtube-dl_formats+json;charset=utf-8");
recordInfo.setEnforceLength(true);
- JsonObject ydlJson = (JsonObject) curi.getData().get("ydlJson");
- StringWriter stringWriter = new StringWriter();
- JsonWriter jsonWriter = new JsonWriter(stringWriter);
- jsonWriter.setIndent(" ");
- Streams.write(ydlJson, jsonWriter);
+ JSONObject ydlJson = (JSONObject) curi.getData().get("ydlJson");
+ String ydlJsonString = ydlJson.toString(1);
- byte[] b = stringWriter.toString().getBytes("UTF-8");
+ byte[] b = ydlJsonString.getBytes("UTF-8");
recordInfo.setContentStream(new ByteArrayInputStream(b));
recordInfo.setContentLength((long) b.length);