lee/xpipe
1
0
Fork 0
mirror of https://github.com/xpipe-io/xpipe.git synced 2026-07-08 11:30:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'transfer-estimate' into 24-release

Browse Source
This commit is contained in:
crschnick
2026-06-20 11:29:04 +00:00
parent 9197bb980f
commit 899140b59d
7 changed files with 128 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/src/main/java/io/xpipe/app/browser/file/BrowserFileSystemSavedState.java
+3 -3
View File
@@ -59,8 +59,8 @@ public class BrowserFileSystemSavedState {
}
static BrowserFileSystemSavedState loadForStore(BrowserFileSystemTabModel model) {
var state = AppCache.getNonNull(
"fs-state-" + model.getEntry().get().getUuid(), BrowserFileSystemSavedState.class, () -> {
var state = AppCache.getNonNullMapEntry(
"fs-state", model.getEntry().get().getUuid().toString(), BrowserFileSystemSavedState.class, () -> {
return new BrowserFileSystemSavedState();
});
state.setModel(model);
@@ -72,7 +72,7 @@ public class BrowserFileSystemSavedState {
return;
}
AppCache.update("fs-state-" + model.getEntry().get().getUuid(), this);
AppCache.updateMapEntry("fs-state", model.getEntry().get().getUuid().toString(), this);
}
public void cd(FilePath dir) {
app/src/main/java/io/xpipe/app/browser/file/BrowserFileTransferOperation.java
+58 -15
View File
@@ -1,10 +1,12 @@
package io.xpipe.app.browser.file;
import io.xpipe.app.core.AppCache;
import io.xpipe.app.core.mode.AppOperationMode;
import io.xpipe.app.ext.FileEntry;
import io.xpipe.app.ext.FileKind;
import io.xpipe.app.ext.FileSystem;
import io.xpipe.app.issue.ErrorEventFactory;
import io.xpipe.app.util.GlobalTimer;
import io.xpipe.app.util.ThreadHelper;
import io.xpipe.app.util.FilePath;
@@ -20,6 +22,7 @@ import java.time.Duration;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
@@ -262,7 +265,7 @@ public class BrowserFileTransferOperation {
// Source might have been deleted meanwhile
var exists = source.getFileSystem().directoryExists(source.getPath());
if (!exists) {
progress.accept(BrowserTransferProgress.finished(source.getName(), 0));
updateProgress(BrowserTransferProgress.finished(source.getName(), 0));
return;
}
@@ -274,7 +277,7 @@ public class BrowserFileTransferOperation {
var baseRelative = source.getPath().getParent().toDirectory();
source.getFileSystem().traverseFilesRecursively(source.getFileSystem(), source.getPath(), fileEntry -> {
if (cancelled()) {
progress.accept(BrowserTransferProgress.finished(source.getName() + " ...", totalSize.get()));
updateProgress(BrowserTransferProgress.finished(source.getName() + " ...", totalSize.get()));
return false;
}
@@ -290,7 +293,7 @@ public class BrowserFileTransferOperation {
// This one is up-to-date and does not need to be recalculated
// If we don't have a size, it doesn't matter that much as the total size is only for display
totalSize.addAndGet(fileEntry.getFileSizeLong().orElse(0));
progress.accept(new BrowserTransferProgress(source.getName() + " ...", 0, totalSize.get()));
updateProgress(new BrowserTransferProgress(source.getName() + " ...", 0, totalSize.get()));
}
return true;
});
@@ -298,7 +301,7 @@ public class BrowserFileTransferOperation {
// Source might have been deleted meanwhile
var exists = source.getFileSystem().fileExists(source.getPath());
if (!exists) {
progress.accept(BrowserTransferProgress.finished(source.getName(), 0));
updateProgress(BrowserTransferProgress.finished(source.getName(), 0));
return;
}
@@ -307,7 +310,7 @@ public class BrowserFileTransferOperation {
totalSize.addAndGet(source.getFileSizeLong().orElse(0));
} else {
// Unsupported type, e.g. a socket
progress.accept(BrowserTransferProgress.finished(source.getName(), 0));
updateProgress(BrowserTransferProgress.finished(source.getName(), 0));
return;
}
@@ -350,7 +353,45 @@ public class BrowserFileTransferOperation {
}
}
transfer(sourceFile.getPath(), optimizedSourceFs, targetFile, targetFs, transferred, totalSize);
var transferRunning = new AtomicBoolean(true);
var sourceId = optimizedSourceFs.getIdentifier();
var targetId = targetFs.getIdentifier();
var fileSize = optimizedSourceFs.getFileSize(sourceFile.getPath());
var startTransferred = transferred.get();
if ((!optimizedSourceFs.hasAccurateProgress() || !targetFs.hasAccurateProgress()) && sourceId.isPresent() && targetId.isPresent()) {
var mapKey = sourceId.get() + "-" + targetId.get();
Long cachedSpeed = AppCache.getNonNullMapEntry("transferSpeedEstimate", mapKey, Long.class, () -> null);
var counter = new AtomicInteger();
if (cachedSpeed != null) {
GlobalTimer.scheduleUntil(Duration.ofMillis(100), false, () -> {
if (!transferRunning.get()) {
return true;
}
// Divide by 9 and not 10 to overreport progress a bit. Better than underreporting
var addedProgress = Math.min(fileSize, (long) counter.incrementAndGet() * cachedSpeed / 9);
updateProgress(
new BrowserTransferProgress(sourceFile.getName(), startTransferred + addedProgress, totalSize.get()));
return false;
});
} else {
updateProgress(new BrowserTransferProgress(sourceFile.getName(), 0, 0));
}
}
var startProgress = new BrowserTransferProgress(source.getName(), transferred.get(), totalSize.get());
try {
transfer(sourceFile.getPath(), optimizedSourceFs, targetFile, targetFs, transferred, totalSize, fileSize);
} finally {
transferRunning.set(false);
}
if (sourceId.isPresent() && targetId.isPresent()) {
var mapKey = sourceId.get() + "-" + targetId.get();
var speed = BrowserTransferProgress.estimateTransferSpeed(List.of(startProgress),
new BrowserTransferProgress(sourceFile.getName(), transferred.get(), totalSize.get()));
AppCache.updateMapEntry("transferSpeedEstimate", mapKey, speed);
}
}
}
} finally {
@@ -407,19 +448,19 @@ public class BrowserFileTransferOperation {
FilePath targetFile,
FileSystem targetFs,
AtomicLong transferred,
AtomicLong totalSize)
AtomicLong totalSize,
long fileSize)
throws Exception {
if (cancelled()) {
return;
}
updateProgress(new BrowserTransferProgress(sourceFile.getFileName(), 0, 0));
var fileSize = sourceFs.getFileSize(sourceFile);
updateProgress(new BrowserTransferProgress(sourceFile.getFileName(), transferred.get(), totalSize.get()));
if (transferInline(sourceFile, sourceFs, targetFile, targetFs) || cancelled()) {
if (!cancelled()) {
updateProgress(BrowserTransferProgress.finished(sourceFile.getFileName(), fileSize));
transferred.addAndGet(fileSize);
updateProgress(new BrowserTransferProgress(sourceFile.getFileName(), transferred.get(), totalSize.get()));
}
return;
}
@@ -442,7 +483,7 @@ public class BrowserFileTransferOperation {
}
outputStream = targetFs.openOutput(targetFile, fileSize);
transferFile(sourceFile, inputStream, outputStream, transferred, totalSize, fileSize);
transferFile(sourceFile, inputStream, outputStream, transferred, totalSize, fileSize, sourceFs.hasAccurateProgress() && targetFs.hasAccurateProgress());
} catch (Exception ex) {
// Mark progress as finished to reset any progress display
updateProgress(BrowserTransferProgress.finished(sourceFile.getFileName(), transferred.get()));
@@ -516,7 +557,8 @@ public class BrowserFileTransferOperation {
OutputStream outputStream,
AtomicLong transferred,
AtomicLong total,
long expectedFileSize)
long expectedFileSize,
boolean reportProgress)
throws Exception {
// Initialize progress immediately prior to reading anything
updateProgress(new BrowserTransferProgress(sourceFile.getFileName(), transferred.get(), total.get()));
@@ -543,8 +585,9 @@ public class BrowserFileTransferOperation {
outputStream.write(buffer, 0, read);
transferred.addAndGet(read);
readCount.addAndGet(read);
updateProgress(
new BrowserTransferProgress(sourceFile.getFileName(), transferred.get(), total.get()));
if (reportProgress) {
updateProgress(new BrowserTransferProgress(sourceFile.getFileName(), transferred.get(), total.get()));
}
}
outputStream.flush();
app/src/main/java/io/xpipe/app/browser/file/BrowserTransferProgress.java
+5
View File
@@ -29,6 +29,11 @@ public class BrowserTransferProgress {
return 0;
}
if (list.size() == 1) {
var ms = Math.max(1, now.getTimestamp().toEpochMilli() - list.getFirst().getTimestamp().toEpochMilli());
return Math.round((now.getTransferred() - list.getFirst().getTransferred()) / (ms / 1000.0));
}
var rSize = list.size() > 1 ? list.size() - 1 : list.size();
var r = new double[rSize];
for (int i = 0; i < rSize; i++) {
app/src/main/java/io/xpipe/app/core/AppCache.java
+34
View File
@@ -1,5 +1,6 @@
package io.xpipe.app.core;
import com.fasterxml.jackson.core.type.TypeReference;
import io.xpipe.app.issue.ErrorEventFactory;
import io.xpipe.app.util.JacksonMapper;
@@ -8,13 +9,16 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.type.TypeFactory;
import lombok.Getter;
import lombok.Setter;
import lombok.val;
import org.apache.commons.io.FileUtils;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
public class AppCache {
@@ -51,6 +55,20 @@ public class AppCache {
return getNonNull(key, TypeFactory.defaultInstance().constructType(type), notPresent);
}
public static <T> T getNonNullMapEntry(String key, String mapKey, Class<?> type, Supplier<T> notPresent) {
return getNonNullMapEntry(key, mapKey, TypeFactory.defaultInstance().constructType(type), notPresent);
}
public static <T> T getNonNullMapEntry(String key, String mapKey, JavaType type, Supplier<T> notPresent) {
var mapType = TypeFactory.defaultInstance().constructMapLikeType(Map.class, TypeFactory.defaultInstance().constructType(String.class), type);
Map<String, T> map = getNonNull(key, mapType, () -> null);
if (map != null) {
return map.get(mapKey);
} else {
return notPresent.get();
}
}
@SuppressWarnings("unchecked")
public static <T> T getNonNull(String key, JavaType type, Supplier<T> notPresent) {
var path = getPath(key);
@@ -104,6 +122,22 @@ public class AppCache {
return notPresent;
}
public static <T> void updateMapEntry(String key, String mapKey, T val) {
if (val == null) {
return;
}
var mapType = TypeFactory.defaultInstance().constructMapLikeType(Map.class, String.class, val.getClass());
Map<String, T> map = getNonNull(key, mapType, () -> null);
if (map != null) {
map.put(mapKey, val);
update(key, map);
} else {
var newMap = Map.of(mapKey, val);
update(key, newMap);
}
}
public static <T> void update(String key, T val) {
var path = getPath(key);
app/src/main/java/io/xpipe/app/ext/ConnectionFileSystem.java
+12
View File
@@ -5,6 +5,7 @@ import io.xpipe.app.process.CommandBuilder;
import io.xpipe.app.process.OsFileSystem;
import io.xpipe.app.process.ShellControl;
import io.xpipe.app.process.ShellDialects;
import io.xpipe.app.storage.DataStorage;
import io.xpipe.app.util.DocumentationLink;
import io.xpipe.app.util.FilePath;
import io.xpipe.app.util.OsType;
@@ -20,6 +21,7 @@ import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Stream;
@Getter
@@ -32,6 +34,11 @@ public class ConnectionFileSystem implements FileSystem {
this.shellControl = shellControl;
}
@Override
public boolean hasAccurateProgress() {
return true;
}
@Override
public boolean writeInstantIfPossible(FileSystem sourceFs, FilePath sourceFile, FilePath targetFile)
throws Exception {
@@ -421,6 +428,11 @@ public class ConnectionFileSystem implements FileSystem {
}
}
@Override
public Optional<UUID> getIdentifier() {
return !shellControl.isLocal() ? shellControl.getSourceStoreId() : Optional.of(DataStorage.LOCAL_ID);
}
@Override
public void close() {
// In case the shell control is already in an invalid state, this operation might fail
app/src/main/java/io/xpipe/app/ext/FileSystem.java
+5
View File
@@ -9,11 +9,14 @@ import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.function.Predicate;
import java.util.stream.Stream;
public interface FileSystem extends Closeable, AutoCloseable {
boolean hasAccurateProgress();
boolean writeInstantIfPossible(FileSystem sourceFs, FilePath sourceFile, FilePath targetFile) throws Exception;
boolean readInstantIfPossible(FilePath sourceFile, FileSystem targetFs, FilePath targetFile) throws Exception;
@@ -141,4 +144,6 @@ public interface FileSystem extends Closeable, AutoCloseable {
List<FilePath> listRoots() throws Exception;
List<FilePath> listCommonDirectories() throws Exception;
Optional<UUID> getIdentifier();
}
app/src/main/java/io/xpipe/app/ext/WrapperFileSystem.java
+11
View File
@@ -9,6 +9,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.function.Supplier;
import java.util.stream.Stream;
@@ -34,6 +35,11 @@ public class WrapperFileSystem implements FileSystem {
consumer.accept(fs);
}
@Override
public boolean hasAccurateProgress() {
return fs.hasAccurateProgress();
}
@Override
public boolean writeInstantIfPossible(FileSystem sourceFs, FilePath sourceFile, FilePath targetFile)
throws Exception {
@@ -378,6 +384,11 @@ public class WrapperFileSystem implements FileSystem {
return fs.listCommonDirectories();
}
@Override
public Optional<UUID> getIdentifier() {
return fs.getIdentifier();
}
@Override
public void close() throws IOException {
fs.close();