lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-22 05:35:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #468 from internetarchive/remove-jna

Browse Source
This commit is contained in:
Alex Osborne
2022-03-03 17:16:34 +09:00
committed by GitHub
parent c6d4251e4f 71b40cf882
commit 69f231b8ab
6 changed files with 20 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/pom.xml
-6
View File
@@ -167,12 +167,6 @@
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
<version>3.2.3</version>
</dependency>
<dependency>
<groupId>org.netpreserve.commons</groupId>
<artifactId>webarchive-commons</artifactId>
commons/src/main/java/org/archive/bdb/BdbModule.java
+9 -7
View File
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.math.BigInteger;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -43,7 +44,6 @@ import org.apache.commons.io.filefilter.IOFileFilter;
import org.archive.checkpointing.Checkpoint;
import org.archive.checkpointing.Checkpointable;
import org.archive.spring.ConfigPath;
import org.archive.util.FilesystemLinkMaker;
import org.archive.util.IdentityCacheable;
import org.archive.util.ObjectIdentityBdbManualCache;
import org.archive.util.ObjectIdentityCache;
@@ -548,8 +548,10 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable, Disposab
filedata[i] += ","+f.length();
if(getUseHardLinkCheckpoints()) {
File hardLink = new File(envCpDir,filedata[i]);
if (!FilesystemLinkMaker.makeHardLink(f.getAbsolutePath(), hardLink.getAbsolutePath())) {
LOGGER.log(Level.SEVERE, "unable to create required checkpoint link "+hardLink);
try {
Files.createLink(hardLink.toPath(), f.toPath().toAbsolutePath());
} catch (IOException | UnsupportedOperationException e) {
LOGGER.log(Level.SEVERE, "unable to create required checkpoint link " + hardLink, e);
}
}
}
@@ -600,10 +602,10 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable, Disposab
LOGGER.log(Level.SEVERE, "unable to delete obstructing file "+destFile);
}
}
boolean status = FilesystemLinkMaker.makeHardLink(cpFile.getAbsolutePath(), destFile.getAbsolutePath());
if (!status) {
LOGGER.log(Level.SEVERE, "unable to create required restore link "+destFile);
try {
Files.createLink(destFile.toPath(), cpFile.toPath().toAbsolutePath());
} catch (IOException | UnsupportedOperationException e) {
LOGGER.log(Level.SEVERE, "unable to create required restore link " + destFile, e);
}
}
commons/src/main/java/org/archive/spring/PathSharingContext.java
+6 -4
View File
@@ -23,6 +23,8 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
@@ -31,7 +33,6 @@ import java.util.logging.Logger;
import org.apache.commons.io.FileUtils;
import org.archive.util.ArchiveUtils;
import org.archive.util.FilesystemLinkMaker;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
import org.springframework.context.ApplicationContext;
@@ -161,9 +162,10 @@ public class PathSharingContext extends FileSystemXmlApplicationContext {
// attempt to symlink "latest" to launch dir
File latestSymlink = new File(getConfigurationFile().getParentFile(), "latest");
latestSymlink.delete();
boolean success = FilesystemLinkMaker.makeSymbolicLink(currentLaunchDir.getName(), latestSymlink.getPath());
if (!success) {
LOGGER.warning("failed to create symlink from " + latestSymlink + " to " + currentLaunchDir);
try {
Files.createSymbolicLink(latestSymlink.toPath(), Paths.get(currentLaunchDir.getName()));
} catch (IOException | UnsupportedOperationException e) {
LOGGER.log(Level.WARNING, "failed to create symlink from " + latestSymlink + " to " + currentLaunchDir, e);
}
} catch (IOException e) {
LOGGER.log(Level.SEVERE, "failed to initialize launch directory: " + e);
commons/src/main/java/org/archive/util/CLibrary.java
-40
View File
@@ -1,40 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
import com.sun.jna.Library;
import com.sun.jna.Native;
import com.sun.jna.Platform;
/**
* Interface to standard C library functions; initially just link().
*
* Via StackOverflow answer:
* http://stackoverflow.com/questions/783075/creating-a-hard-link-in-java/3023349#3023349
*
*/
public interface CLibrary extends Library {
CLibrary INSTANCE = (CLibrary)
Native.loadLibrary((Platform.isWindows() ? "msvcrt" : "c"),
CLibrary.class);
int link(String existingPath, String newPath);
int symlink(String existingPath, String newPath);
}
commons/src/main/java/org/archive/util/FilesystemLinkMaker.java
-139
View File
@@ -1,139 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
import java.io.File;
import java.io.IOException;
import java.util.logging.Logger;
import com.sun.jna.Native;
import com.sun.jna.Platform;
import com.sun.jna.Pointer;
import com.sun.jna.Structure;
import com.sun.jna.win32.StdCallLibrary;
/**
* Wrapper for platform-dependent hard link creation.
*
* @see <a href="http://stackoverflow.com/questions/783075/creating-a-hard-link-in-java/3023349#3023349">http://stackoverflow.com/questions/783075/creating-a-hard-link-in-java/3023349#3023349</a>
*/
public class FilesystemLinkMaker {
private static final Logger logger = Logger.getLogger(FilesystemLinkMaker.class.getName());
// see https://github.com/twall/jna/blob/master/www/GettingStarted.md
public interface Kernel32Library extends StdCallLibrary {
Kernel32Library INSTANCE = (Kernel32Library) (Platform.isWindows()
? Native.loadLibrary("kernel32", Kernel32Library.class)
: null);
/* http://msdn.microsoft.com/en-us/library/aa379560%28VS.85%29.aspx
* http://en.wikipedia.org/wiki/Java_Native_Access
*
* typedef struct _SECURITY_ATTRIBUTES {
* DWORD nLength;
* LPVOID lpSecurityDescriptor;
* BOOL bInheritHandle;
* } SECURITY_ATTRIBUTES, *PSECURITY_ATTRIBUTES, *LPSECURITY_ATTRIBUTES;
*/
public static class LPSECURITY_ATTRIBUTES extends Structure {
public int nLength;
public Pointer lpSecurityDescriptor;
public boolean bInheritHandle;
}
/*
* http://msdn.microsoft.com/en-us/library/aa363860%28VS.85%29.aspx
* CreateHardLink is a macro that maps to CreateHardLinkA (ansi) or
* CreateHardLinkW (unicode). In initial testing CreateHardLinkW()
* worked, but then it stopped working... go figure.
*/
boolean CreateHardLinkA(String newPath, String existingPath, LPSECURITY_ATTRIBUTES lpSecurityAttributes);
// boolean CreateHardLinkW(String newPath, String existingPath, LPSECURITY_ATTRIBUTES lpSecurityAttributes);
// http://msdn.microsoft.com/en-us/library/aa363866%28v=VS.85%29.aspx
boolean CreateSymbolicLinkA(String newPath, String existingPath, LPSECURITY_ATTRIBUTES lpSecurityAttributes);
}
/**
* Wrapper over platform-dependent system calls to create a hard link.
*
* @return true on success
*/
// XXX could handle errors better (examine errno, throw exception...)
public static boolean makeHardLink(String existingPath, String newPath) {
try {
if (Platform.isWindows()) {
return Kernel32Library.INSTANCE.CreateHardLinkA(newPath, existingPath, null);
} else {
int status = CLibrary.INSTANCE.link(existingPath, newPath);
return status == 0;
}
} catch (UnsatisfiedLinkError e) {
// see https://webarchive.jira.com/browse/HER-1979
logger.warning("hard links not supported on this platform - " + e);
return false;
}
}
/**
* Wrapper over platform-dependent system calls to create a symbolic link.
*
* @return true on success
*/
// XXX could handle errors better (examine errno, throw exception...)
public static boolean makeSymbolicLink(String existingPath, String newPath) {
try {
if (Platform.isWindows()) {
return Kernel32Library.INSTANCE.CreateSymbolicLinkA(newPath, existingPath, null);
} else {
int status = CLibrary.INSTANCE.symlink(existingPath, newPath);
return status == 0;
}
} catch (UnsatisfiedLinkError e) {
// see https://webarchive.jira.com/browse/HER-1979
logger.warning("symbolic links not supported on this platform - " + e);
return false;
}
}
public static void main(String[] args) throws IOException {
File existingPath = File.createTempFile("heritrixHardLinkTestExistingFile", ".tmp");
File newPath = File.createTempFile("heritrixHardLinkTestNewFile", ".tmp");
newPath.delete();
if (FilesystemLinkMaker.makeHardLink(existingPath.getAbsolutePath(), newPath.getAbsolutePath())) {
System.out.println("success - made hard link from " + newPath.getAbsolutePath() + " to " + existingPath.getAbsolutePath());
} else {
System.out.println("failed to make hard link from " + newPath.getAbsolutePath() + " to " + existingPath.getAbsolutePath());
}
existingPath = File.createTempFile("heritrixSymlinkTestExistingFile", ".tmp");
newPath = File.createTempFile("heritrixSymlinkTestNewFile", ".tmp");
newPath.delete();
if (FilesystemLinkMaker.makeSymbolicLink(existingPath.getPath(), newPath.getPath())) {
System.out.println("success - made symlink from " + newPath.getAbsolutePath() + " to " + existingPath.getAbsolutePath());
} else {
System.out.println("failed to make symlink from " + newPath.getAbsolutePath() + " to " + existingPath.getAbsolutePath());
}
}
}
engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java
+5 -4
View File
@@ -24,6 +24,7 @@ import java.io.FileFilter;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -42,7 +43,6 @@ import org.apache.commons.lang.StringUtils;
import org.archive.modules.seeds.SeedModule;
import org.archive.spring.ConfigPath;
import org.archive.util.ArchiveUtils;
import org.archive.util.FilesystemLinkMaker;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
@@ -266,9 +266,10 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn
if (!actionDoneDirFile.equals(doneDir.getFile())) {
actionDoneDirFile.mkdirs();
File doneSymlinkFile = new File(actionDoneDirFile, doneFile.getName());
boolean success = FilesystemLinkMaker.makeSymbolicLink(doneFile.getPath(), doneSymlinkFile.getPath());
if (!success) {
LOGGER.warning("failed to create symlink from " + doneSymlinkFile + " to " + doneFile);
try {
Files.createSymbolicLink(doneSymlinkFile.toPath(), doneFile.toPath());
} catch (IOException | UnsupportedOperationException e) {
LOGGER.log(Level.WARNING, "failed to create symlink from " + doneSymlinkFile + " to " + doneFile, e);
}
}
} catch (IOException e) {