Add support for resuming file upload/download

Pedro Fonseca · Pedro Fonseca · commit 17d86bfb8e16 · 2021-08-27T16:56:48.000+02:00
diff --git a/src/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_EndLStatThrowsSshException.cs b/src/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_EndLStatThrowsSshException.cs
@@ -59,7 +59,7 @@ private void SetupMocks()
                             .Setup(p => p.EndLStat(_statAsyncResult))
                             .Throws(new SshException());
             _sftpSessionMock.InSequence(seq)
-                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 3, null))
+                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 3, null, 0))
                             .Returns(_sftpFileReaderMock.Object);
         }
 
diff --git a/src/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsAlmostSixTimesGreaterThanChunkSize.cs b/src/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsAlmostSixTimesGreaterThanChunkSize.cs
@@ -63,7 +63,7 @@ private void SetupMocks()
                             .Setup(p => p.EndLStat(_statAsyncResult))
                             .Returns(_fileAttributes);
             _sftpSessionMock.InSequence(seq)
-                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize))
+                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize, 0))
                             .Returns(_sftpFileReaderMock.Object);
         }
 
diff --git a/src/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsEqualToChunkSize.cs b/src/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsEqualToChunkSize.cs
@@ -63,7 +63,7 @@ private void SetupMocks()
                             .Setup(p => p.EndLStat(_statAsyncResult))
                             .Returns(_fileAttributes);
             _sftpSessionMock.InSequence(seq)
-                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 2, _fileSize))
+                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 2, _fileSize, 0))
                             .Returns(_sftpFileReaderMock.Object);
         }
 
diff --git a/src/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsExactlyFiveTimesGreaterThanChunkSize.cs b/src/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsExactlyFiveTimesGreaterThanChunkSize.cs
@@ -63,7 +63,7 @@ private void SetupMocks()
                             .Setup(p => p.EndLStat(_statAsyncResult))
                             .Returns(_fileAttributes);
             _sftpSessionMock.InSequence(seq)
-                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 6, _fileSize))
+                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 6, _fileSize, 0))
                             .Returns(_sftpFileReaderMock.Object);
         }
 
diff --git a/src/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsLessThanChunkSize.cs b/src/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsLessThanChunkSize.cs
@@ -63,7 +63,7 @@ private void SetupMocks()
                             .Setup(p => p.EndLStat(_statAsyncResult))
                             .Returns(_fileAttributes);
             _sftpSessionMock.InSequence(seq)
-                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 2, _fileSize))
+                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 2, _fileSize, 0))
                             .Returns(_sftpFileReaderMock.Object);
         }
 
diff --git a/src/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsLittleMoreThanFiveTimesGreaterThanChunkSize.cs b/src/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsLittleMoreThanFiveTimesGreaterThanChunkSize.cs
@@ -63,7 +63,7 @@ private void SetupMocks()
                             .Setup(p => p.EndLStat(_statAsyncResult))
                             .Returns(_fileAttributes);
             _sftpSessionMock.InSequence(seq)
-                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize))
+                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize, 0))
                             .Returns(_sftpFileReaderMock.Object);
         }
 
diff --git a/src/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanTenTimesGreaterThanChunkSize.cs b/src/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanTenTimesGreaterThanChunkSize.cs
@@ -63,7 +63,7 @@ private void SetupMocks()
                             .Setup(p => p.EndLStat(_statAsyncResult))
                             .Returns(_fileAttributes);
             _sftpSessionMock.InSequence(seq)
-                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 10, _fileSize))
+                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 10, _fileSize, 0))
                             .Returns(_sftpFileReaderMock.Object);
         }
 
diff --git a/src/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsZero.cs b/src/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsZero.cs
@@ -63,7 +63,7 @@ private void SetupMocks()
                             .Setup(p => p.EndLStat(_statAsyncResult))
                             .Returns(_fileAttributes);
             _sftpSessionMock.InSequence(seq)
-                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 1, _fileSize))
+                            .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 1, _fileSize, 0))
                             .Returns(_sftpFileReaderMock.Object);
         }
 
diff --git a/src/Renci.SshNet/IServiceFactory.cs b/src/Renci.SshNet/IServiceFactory.cs
@@ -64,7 +64,7 @@ internal partial interface IServiceFactory
         /// <exception cref="SshConnectionException">No key exchange algorithm is supported by both client and server.</exception>
         IKeyExchange CreateKeyExchange(IDictionary<string, Type> clientAlgorithms, string[] serverAlgorithms);
 
-        ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize);
+        ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize, ulong offset = 0);
 
         ISftpResponseFactory CreateSftpResponseFactory();
 
diff --git a/src/Renci.SshNet/ServiceFactory.cs b/src/Renci.SshNet/ServiceFactory.cs
@@ -110,7 +110,7 @@ from s in serverAlgorithms
             return keyExchangeAlgorithmType.CreateInstance<IKeyExchange>();
         }
 
-        public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize)
+        public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize, ulong offset = 0)
         {
             const int defaultMaxPendingReads = 3;
 
@@ -132,7 +132,7 @@ public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSe
             {
                 var fileAttributes = sftpSession.EndLStat(statAsyncResult);
                 fileSize = fileAttributes.Size;
-                maxPendingReads = Math.Min(10, (int) Math.Ceiling((double) fileAttributes.Size / chunkSize) + 1);
+                maxPendingReads = Math.Min(10, (int) Math.Ceiling((double)(fileSize - (long)offset) / chunkSize) + 1);
             }
             catch (SshException ex)
             {
@@ -142,7 +142,7 @@ public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSe
                 DiagnosticAbstraction.Log(string.Format("Failed to obtain size of file. Allowing maximum {0} pending reads: {1}", maxPendingReads, ex));
             }
 
-            return sftpSession.CreateFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize);
+            return sftpSession.CreateFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize, offset);
         }
 
         public ISftpResponseFactory CreateSftpResponseFactory()
diff --git a/src/Renci.SshNet/Sftp/ISftpSession.cs b/src/Renci.SshNet/Sftp/ISftpSession.cs
@@ -341,6 +341,6 @@ void RequestWrite(byte[] handle,
         /// </remarks>
         uint CalculateOptimalWriteLength(uint bufferSize, byte[] handle);
 
-        ISftpFileReader CreateFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize);
+        ISftpFileReader CreateFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize, ulong offset = 0);
     }
 }
diff --git a/src/Renci.SshNet/Sftp/SftpFileReader.cs b/src/Renci.SshNet/Sftp/SftpFileReader.cs
@@ -53,12 +53,14 @@ internal class SftpFileReader : ISftpFileReader
         /// <param name="chunkSize">The size of a individual read-ahead chunk.</param>
         /// <param name="maxPendingReads">The maximum number of pending reads.</param>
         /// <param name="fileSize">The size of the file, if known; otherwise, <c>null</c>.</param>
-        public SftpFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize)
+        /// <param name="offset">The offset to resume from</param>
+        public SftpFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize, ulong offset = 0)
         {
             _handle = handle;
             _sftpSession = sftpSession;
             _chunkSize = chunkSize;
             _fileSize = fileSize;
+            _offset = _readAheadOffset = offset;
             _semaphore = new SemaphoreLight(maxPendingReads);
             _queue = new Dictionary<int, BufferedRead>(maxPendingReads);
             _readLock = new object();
diff --git a/src/Renci.SshNet/Sftp/SftpSession.cs b/src/Renci.SshNet/Sftp/SftpSession.cs
@@ -136,9 +136,9 @@ public string GetCanonicalPath(string path)
             return string.Format(CultureInfo.InvariantCulture, "{0}{1}{2}", canonizedPath, slash, pathParts[pathParts.Length - 1]);
         }
 
-        public ISftpFileReader CreateFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize)
+        public ISftpFileReader CreateFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize, ulong offset = 0)
         {
-            return new SftpFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize);
+            return new SftpFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize, offset);
         }
 
         internal string GetFullRemotePath(string path)
diff --git a/src/Renci.SshNet/SftpClient.cs b/src/Renci.SshNet/SftpClient.cs
@@ -801,10 +801,15 @@ public void UploadFile(Stream input, string path, bool canOverride, Action<ulong
 
             var flags = Flags.Write | Flags.Truncate;
 
-            if (canOverride)
-                flags |= Flags.CreateNewOrOpen;
+            if (input.Position > 0)
+                flags = Flags.Write | Flags.Append;
             else
-                flags |= Flags.CreateNew;
+            {
+                if (canOverride)
+                    flags |= Flags.CreateNewOrOpen;
+                else
+                    flags |= Flags.CreateNew;
+            }
 
             InternalUploadFile(input, path, flags, null, uploadCallback);
         }
@@ -931,10 +936,15 @@ public IAsyncResult BeginUploadFile(Stream input, string path, bool canOverride,
 
             var flags = Flags.Write | Flags.Truncate;
 
-            if (canOverride)
-                flags |= Flags.CreateNewOrOpen;
+            if (input.Position > 0)
+                flags = Flags.Write | Flags.Append;
             else
-                flags |= Flags.CreateNew;
+            {
+                if (canOverride)
+                    flags |= Flags.CreateNewOrOpen;
+                else
+                    flags |= Flags.CreateNew;
+            }
 
             var asyncResult = new SftpUploadAsyncResult(asyncCallback, state);
 
@@ -2050,7 +2060,7 @@ private void InternalDownloadFile(string path, Stream output, SftpDownloadAsyncR
 
             var fullPath = _sftpSession.GetCanonicalPath(path);
 
-            using (var fileReader = ServiceFactory.CreateSftpFileReader(fullPath, _sftpSession, _bufferSize))
+            using (var fileReader = ServiceFactory.CreateSftpFileReader(fullPath, _sftpSession, _bufferSize, (ulong)output.Position))
             {
                 var totalBytesRead = 0UL;
 
@@ -2106,7 +2116,7 @@ private void InternalUploadFile(Stream input, string path, Flags flags, SftpUplo
 
             var handle = _sftpSession.RequestOpen(fullPath, flags);
 
-            ulong offset = 0;
+            ulong offset = (ulong)input.Position;
 
             // create buffer of optimal length
             var buffer = new byte[_sftpSession.CalculateOptimalWriteLength(_bufferSize, handle)];

Original file line number	Diff line number	Diff line change
`@@ -59,7 +59,7 @@ private void SetupMocks()`
`59`	`59`	`.Setup(p => p.EndLStat(_statAsyncResult))`
`60`	`60`	`.Throws(new SshException());`
`61`	`61`	`_sftpSessionMock.InSequence(seq)`
`62`		`- .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 3, null))`
	`62`	`+ .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 3, null, 0))`
`63`	`63`	`.Returns(_sftpFileReaderMock.Object);`
`64`	`64`	`}`
`65`	`65`
Original file line number	Diff line number	Diff line change
`@@ -63,7 +63,7 @@ private void SetupMocks()`
`63`	`63`	`.Setup(p => p.EndLStat(_statAsyncResult))`
`64`	`64`	`.Returns(_fileAttributes);`
`65`	`65`	`_sftpSessionMock.InSequence(seq)`
`66`		`- .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize))`
	`66`	`+ .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize, 0))`
`67`	`67`	`.Returns(_sftpFileReaderMock.Object);`
`68`	`68`	`}`
`69`	`69`
Original file line number	Diff line number	Diff line change
`@@ -110,7 +110,7 @@ from s in serverAlgorithms`
`110`	`110`	`return keyExchangeAlgorithmType.CreateInstance<IKeyExchange>();`
`111`	`111`	`}`
`112`	`112`
`113`		`- public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize)`
	`113`	`+ public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize, ulong offset = 0)`
`114`	`114`	`{`
`115`	`115`	`const int defaultMaxPendingReads = 3;`
`116`	`116`
`@@ -132,7 +132,7 @@ public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSe`
`132`	`132`	`{`
`133`	`133`	`var fileAttributes = sftpSession.EndLStat(statAsyncResult);`
`134`	`134`	`fileSize = fileAttributes.Size;`
`135`		`- maxPendingReads = Math.Min(10, (int) Math.Ceiling((double) fileAttributes.Size / chunkSize) + 1);`
	`135`	`+ maxPendingReads = Math.Min(10, (int) Math.Ceiling((double)(fileSize - (long)offset) / chunkSize) + 1);`
`136`	`136`	`}`
`137`	`137`	`catch (SshException ex)`
`138`	`138`	`{`
`@@ -142,7 +142,7 @@ public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSe`
`142`	`142`	`DiagnosticAbstraction.Log(string.Format("Failed to obtain size of file. Allowing maximum {0} pending reads: {1}", maxPendingReads, ex));`
`143`	`143`	`}`
`144`	`144`
`145`		`- return sftpSession.CreateFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize);`
	`145`	`+ return sftpSession.CreateFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize, offset);`
`146`	`146`	`}`
`147`	`147`
`148`	`148`	`public ISftpResponseFactory CreateSftpResponseFactory()`