From 899140b59ded5e11f92e927ffe18f4240755d239 Mon Sep 17 00:00:00 2001 From: crschnick Date: Sat, 20 Jun 2026 11:29:04 +0000 Subject: [PATCH] Merge branch 'transfer-estimate' into 24-release --- .../file/BrowserFileSystemSavedState.java | 6 +- .../file/BrowserFileTransferOperation.java | 73 +++++++++++++++---- .../browser/file/BrowserTransferProgress.java | 5 ++ .../main/java/io/xpipe/app/core/AppCache.java | 34 +++++++++ .../xpipe/app/ext/ConnectionFileSystem.java | 12 +++ .../java/io/xpipe/app/ext/FileSystem.java | 5 ++ .../io/xpipe/app/ext/WrapperFileSystem.java | 11 +++ 7 files changed, 128 insertions(+), 18 deletions(-) diff --git a/app/src/main/java/io/xpipe/app/browser/file/BrowserFileSystemSavedState.java b/app/src/main/java/io/xpipe/app/browser/file/BrowserFileSystemSavedState.java index cbc702eda..4d98e97a4 100644 --- a/app/src/main/java/io/xpipe/app/browser/file/BrowserFileSystemSavedState.java +++ b/app/src/main/java/io/xpipe/app/browser/file/BrowserFileSystemSavedState.java @@ -59,8 +59,8 @@ public class BrowserFileSystemSavedState { } static BrowserFileSystemSavedState loadForStore(BrowserFileSystemTabModel model) { - var state = AppCache.getNonNull( - "fs-state-" + model.getEntry().get().getUuid(), BrowserFileSystemSavedState.class, () -> { + var state = AppCache.getNonNullMapEntry( + "fs-state", model.getEntry().get().getUuid().toString(), BrowserFileSystemSavedState.class, () -> { return new BrowserFileSystemSavedState(); }); state.setModel(model); @@ -72,7 +72,7 @@ public class BrowserFileSystemSavedState { return; } - AppCache.update("fs-state-" + model.getEntry().get().getUuid(), this); + AppCache.updateMapEntry("fs-state", model.getEntry().get().getUuid().toString(), this); } public void cd(FilePath dir) { diff --git a/app/src/main/java/io/xpipe/app/browser/file/BrowserFileTransferOperation.java b/app/src/main/java/io/xpipe/app/browser/file/BrowserFileTransferOperation.java index 6c06033b2..a38b782c7 100644 --- a/app/src/main/java/io/xpipe/app/browser/file/BrowserFileTransferOperation.java +++ b/app/src/main/java/io/xpipe/app/browser/file/BrowserFileTransferOperation.java @@ -1,10 +1,12 @@ package io.xpipe.app.browser.file; +import io.xpipe.app.core.AppCache; import io.xpipe.app.core.mode.AppOperationMode; import io.xpipe.app.ext.FileEntry; import io.xpipe.app.ext.FileKind; import io.xpipe.app.ext.FileSystem; import io.xpipe.app.issue.ErrorEventFactory; +import io.xpipe.app.util.GlobalTimer; import io.xpipe.app.util.ThreadHelper; import io.xpipe.app.util.FilePath; @@ -20,6 +22,7 @@ import java.time.Duration; import java.util.LinkedHashMap; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -262,7 +265,7 @@ public class BrowserFileTransferOperation { // Source might have been deleted meanwhile var exists = source.getFileSystem().directoryExists(source.getPath()); if (!exists) { - progress.accept(BrowserTransferProgress.finished(source.getName(), 0)); + updateProgress(BrowserTransferProgress.finished(source.getName(), 0)); return; } @@ -274,7 +277,7 @@ public class BrowserFileTransferOperation { var baseRelative = source.getPath().getParent().toDirectory(); source.getFileSystem().traverseFilesRecursively(source.getFileSystem(), source.getPath(), fileEntry -> { if (cancelled()) { - progress.accept(BrowserTransferProgress.finished(source.getName() + " ...", totalSize.get())); + updateProgress(BrowserTransferProgress.finished(source.getName() + " ...", totalSize.get())); return false; } @@ -290,7 +293,7 @@ public class BrowserFileTransferOperation { // This one is up-to-date and does not need to be recalculated // If we don't have a size, it doesn't matter that much as the total size is only for display totalSize.addAndGet(fileEntry.getFileSizeLong().orElse(0)); - progress.accept(new BrowserTransferProgress(source.getName() + " ...", 0, totalSize.get())); + updateProgress(new BrowserTransferProgress(source.getName() + " ...", 0, totalSize.get())); } return true; }); @@ -298,7 +301,7 @@ public class BrowserFileTransferOperation { // Source might have been deleted meanwhile var exists = source.getFileSystem().fileExists(source.getPath()); if (!exists) { - progress.accept(BrowserTransferProgress.finished(source.getName(), 0)); + updateProgress(BrowserTransferProgress.finished(source.getName(), 0)); return; } @@ -307,7 +310,7 @@ public class BrowserFileTransferOperation { totalSize.addAndGet(source.getFileSizeLong().orElse(0)); } else { // Unsupported type, e.g. a socket - progress.accept(BrowserTransferProgress.finished(source.getName(), 0)); + updateProgress(BrowserTransferProgress.finished(source.getName(), 0)); return; } @@ -350,7 +353,45 @@ public class BrowserFileTransferOperation { } } - transfer(sourceFile.getPath(), optimizedSourceFs, targetFile, targetFs, transferred, totalSize); + var transferRunning = new AtomicBoolean(true); + var sourceId = optimizedSourceFs.getIdentifier(); + var targetId = targetFs.getIdentifier(); + var fileSize = optimizedSourceFs.getFileSize(sourceFile.getPath()); + var startTransferred = transferred.get(); + if ((!optimizedSourceFs.hasAccurateProgress() || !targetFs.hasAccurateProgress()) && sourceId.isPresent() && targetId.isPresent()) { + var mapKey = sourceId.get() + "-" + targetId.get(); + Long cachedSpeed = AppCache.getNonNullMapEntry("transferSpeedEstimate", mapKey, Long.class, () -> null); + var counter = new AtomicInteger(); + if (cachedSpeed != null) { + GlobalTimer.scheduleUntil(Duration.ofMillis(100), false, () -> { + if (!transferRunning.get()) { + return true; + } + + // Divide by 9 and not 10 to overreport progress a bit. Better than underreporting + var addedProgress = Math.min(fileSize, (long) counter.incrementAndGet() * cachedSpeed / 9); + updateProgress( + new BrowserTransferProgress(sourceFile.getName(), startTransferred + addedProgress, totalSize.get())); + return false; + }); + } else { + updateProgress(new BrowserTransferProgress(sourceFile.getName(), 0, 0)); + } + } + + var startProgress = new BrowserTransferProgress(source.getName(), transferred.get(), totalSize.get()); + try { + transfer(sourceFile.getPath(), optimizedSourceFs, targetFile, targetFs, transferred, totalSize, fileSize); + } finally { + transferRunning.set(false); + } + + if (sourceId.isPresent() && targetId.isPresent()) { + var mapKey = sourceId.get() + "-" + targetId.get(); + var speed = BrowserTransferProgress.estimateTransferSpeed(List.of(startProgress), + new BrowserTransferProgress(sourceFile.getName(), transferred.get(), totalSize.get())); + AppCache.updateMapEntry("transferSpeedEstimate", mapKey, speed); + } } } } finally { @@ -407,19 +448,19 @@ public class BrowserFileTransferOperation { FilePath targetFile, FileSystem targetFs, AtomicLong transferred, - AtomicLong totalSize) + AtomicLong totalSize, + long fileSize) throws Exception { if (cancelled()) { return; } - updateProgress(new BrowserTransferProgress(sourceFile.getFileName(), 0, 0)); - - var fileSize = sourceFs.getFileSize(sourceFile); + updateProgress(new BrowserTransferProgress(sourceFile.getFileName(), transferred.get(), totalSize.get())); if (transferInline(sourceFile, sourceFs, targetFile, targetFs) || cancelled()) { if (!cancelled()) { - updateProgress(BrowserTransferProgress.finished(sourceFile.getFileName(), fileSize)); + transferred.addAndGet(fileSize); + updateProgress(new BrowserTransferProgress(sourceFile.getFileName(), transferred.get(), totalSize.get())); } return; } @@ -442,7 +483,7 @@ public class BrowserFileTransferOperation { } outputStream = targetFs.openOutput(targetFile, fileSize); - transferFile(sourceFile, inputStream, outputStream, transferred, totalSize, fileSize); + transferFile(sourceFile, inputStream, outputStream, transferred, totalSize, fileSize, sourceFs.hasAccurateProgress() && targetFs.hasAccurateProgress()); } catch (Exception ex) { // Mark progress as finished to reset any progress display updateProgress(BrowserTransferProgress.finished(sourceFile.getFileName(), transferred.get())); @@ -516,7 +557,8 @@ public class BrowserFileTransferOperation { OutputStream outputStream, AtomicLong transferred, AtomicLong total, - long expectedFileSize) + long expectedFileSize, + boolean reportProgress) throws Exception { // Initialize progress immediately prior to reading anything updateProgress(new BrowserTransferProgress(sourceFile.getFileName(), transferred.get(), total.get())); @@ -543,8 +585,9 @@ public class BrowserFileTransferOperation { outputStream.write(buffer, 0, read); transferred.addAndGet(read); readCount.addAndGet(read); - updateProgress( - new BrowserTransferProgress(sourceFile.getFileName(), transferred.get(), total.get())); + if (reportProgress) { + updateProgress(new BrowserTransferProgress(sourceFile.getFileName(), transferred.get(), total.get())); + } } outputStream.flush(); diff --git a/app/src/main/java/io/xpipe/app/browser/file/BrowserTransferProgress.java b/app/src/main/java/io/xpipe/app/browser/file/BrowserTransferProgress.java index 8fb8b20dd..9d933de2f 100644 --- a/app/src/main/java/io/xpipe/app/browser/file/BrowserTransferProgress.java +++ b/app/src/main/java/io/xpipe/app/browser/file/BrowserTransferProgress.java @@ -29,6 +29,11 @@ public class BrowserTransferProgress { return 0; } + if (list.size() == 1) { + var ms = Math.max(1, now.getTimestamp().toEpochMilli() - list.getFirst().getTimestamp().toEpochMilli()); + return Math.round((now.getTransferred() - list.getFirst().getTransferred()) / (ms / 1000.0)); + } + var rSize = list.size() > 1 ? list.size() - 1 : list.size(); var r = new double[rSize]; for (int i = 0; i < rSize; i++) { diff --git a/app/src/main/java/io/xpipe/app/core/AppCache.java b/app/src/main/java/io/xpipe/app/core/AppCache.java index 9cb95a8f6..b8df93ebb 100644 --- a/app/src/main/java/io/xpipe/app/core/AppCache.java +++ b/app/src/main/java/io/xpipe/app/core/AppCache.java @@ -1,5 +1,6 @@ package io.xpipe.app.core; +import com.fasterxml.jackson.core.type.TypeReference; import io.xpipe.app.issue.ErrorEventFactory; import io.xpipe.app.util.JacksonMapper; @@ -8,13 +9,16 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.type.TypeFactory; import lombok.Getter; import lombok.Setter; +import lombok.val; import org.apache.commons.io.FileUtils; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; +import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.Supplier; public class AppCache { @@ -51,6 +55,20 @@ public class AppCache { return getNonNull(key, TypeFactory.defaultInstance().constructType(type), notPresent); } + public static T getNonNullMapEntry(String key, String mapKey, Class type, Supplier notPresent) { + return getNonNullMapEntry(key, mapKey, TypeFactory.defaultInstance().constructType(type), notPresent); + } + + public static T getNonNullMapEntry(String key, String mapKey, JavaType type, Supplier notPresent) { + var mapType = TypeFactory.defaultInstance().constructMapLikeType(Map.class, TypeFactory.defaultInstance().constructType(String.class), type); + Map map = getNonNull(key, mapType, () -> null); + if (map != null) { + return map.get(mapKey); + } else { + return notPresent.get(); + } + } + @SuppressWarnings("unchecked") public static T getNonNull(String key, JavaType type, Supplier notPresent) { var path = getPath(key); @@ -104,6 +122,22 @@ public class AppCache { return notPresent; } + public static void updateMapEntry(String key, String mapKey, T val) { + if (val == null) { + return; + } + + var mapType = TypeFactory.defaultInstance().constructMapLikeType(Map.class, String.class, val.getClass()); + Map map = getNonNull(key, mapType, () -> null); + if (map != null) { + map.put(mapKey, val); + update(key, map); + } else { + var newMap = Map.of(mapKey, val); + update(key, newMap); + } + } + public static void update(String key, T val) { var path = getPath(key); diff --git a/app/src/main/java/io/xpipe/app/ext/ConnectionFileSystem.java b/app/src/main/java/io/xpipe/app/ext/ConnectionFileSystem.java index bee48614a..81f06ef07 100644 --- a/app/src/main/java/io/xpipe/app/ext/ConnectionFileSystem.java +++ b/app/src/main/java/io/xpipe/app/ext/ConnectionFileSystem.java @@ -5,6 +5,7 @@ import io.xpipe.app.process.CommandBuilder; import io.xpipe.app.process.OsFileSystem; import io.xpipe.app.process.ShellControl; import io.xpipe.app.process.ShellDialects; +import io.xpipe.app.storage.DataStorage; import io.xpipe.app.util.DocumentationLink; import io.xpipe.app.util.FilePath; import io.xpipe.app.util.OsType; @@ -20,6 +21,7 @@ import java.nio.file.Files; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.stream.Stream; @Getter @@ -32,6 +34,11 @@ public class ConnectionFileSystem implements FileSystem { this.shellControl = shellControl; } + @Override + public boolean hasAccurateProgress() { + return true; + } + @Override public boolean writeInstantIfPossible(FileSystem sourceFs, FilePath sourceFile, FilePath targetFile) throws Exception { @@ -421,6 +428,11 @@ public class ConnectionFileSystem implements FileSystem { } } + @Override + public Optional getIdentifier() { + return !shellControl.isLocal() ? shellControl.getSourceStoreId() : Optional.of(DataStorage.LOCAL_ID); + } + @Override public void close() { // In case the shell control is already in an invalid state, this operation might fail diff --git a/app/src/main/java/io/xpipe/app/ext/FileSystem.java b/app/src/main/java/io/xpipe/app/ext/FileSystem.java index 96c167c88..728023a06 100644 --- a/app/src/main/java/io/xpipe/app/ext/FileSystem.java +++ b/app/src/main/java/io/xpipe/app/ext/FileSystem.java @@ -9,11 +9,14 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.function.Predicate; import java.util.stream.Stream; public interface FileSystem extends Closeable, AutoCloseable { + boolean hasAccurateProgress(); + boolean writeInstantIfPossible(FileSystem sourceFs, FilePath sourceFile, FilePath targetFile) throws Exception; boolean readInstantIfPossible(FilePath sourceFile, FileSystem targetFs, FilePath targetFile) throws Exception; @@ -141,4 +144,6 @@ public interface FileSystem extends Closeable, AutoCloseable { List listRoots() throws Exception; List listCommonDirectories() throws Exception; + + Optional getIdentifier(); } diff --git a/app/src/main/java/io/xpipe/app/ext/WrapperFileSystem.java b/app/src/main/java/io/xpipe/app/ext/WrapperFileSystem.java index a00ebea24..6f3b92514 100644 --- a/app/src/main/java/io/xpipe/app/ext/WrapperFileSystem.java +++ b/app/src/main/java/io/xpipe/app/ext/WrapperFileSystem.java @@ -9,6 +9,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.function.Supplier; import java.util.stream.Stream; @@ -34,6 +35,11 @@ public class WrapperFileSystem implements FileSystem { consumer.accept(fs); } + @Override + public boolean hasAccurateProgress() { + return fs.hasAccurateProgress(); + } + @Override public boolean writeInstantIfPossible(FileSystem sourceFs, FilePath sourceFile, FilePath targetFile) throws Exception { @@ -378,6 +384,11 @@ public class WrapperFileSystem implements FileSystem { return fs.listCommonDirectories(); } + @Override + public Optional getIdentifier() { + return fs.getIdentifier(); + } + @Override public void close() throws IOException { fs.close();