Skip to content

Commit c1b6b9c

Browse files
znamenapznamenap
committed
Fix #395 : SftpClient Enumerates Rather Than Accumulates Directory Items
1 parent cefdc20 commit c1b6b9c

File tree

4 files changed

+347
-18
lines changed

4 files changed

+347
-18
lines changed

Diff for: ‎src/Renci.SshNet.Tests/Classes/SftpClientTest.EnumerateDirectory.cs

+209
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
using Microsoft.VisualStudio.TestTools.UnitTesting;
2+
using Renci.SshNet.Common;
3+
using Renci.SshNet.Tests.Common;
4+
using Renci.SshNet.Tests.Properties;
5+
using System;
6+
using System.Diagnostics;
7+
using System.Linq;
8+
9+
namespace Renci.SshNet.Tests.Classes
10+
{
11+
/// <summary>
12+
/// Implementation of the SSH File Transfer Protocol (SFTP) over SSH.
13+
/// </summary>
14+
public partial class SftpClientTest : TestBase
15+
{
16+
[TestMethod]
17+
[TestCategory("Sftp")]
18+
[ExpectedException(typeof(SshConnectionException))]
19+
public void Test_Sftp_EnumerateDirectory_Without_Connecting()
20+
{
21+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
22+
{
23+
var files = sftp.EnumerateDirectory(".");
24+
foreach (var file in files)
25+
{
26+
Debug.WriteLine(file.FullName);
27+
}
28+
}
29+
}
30+
31+
[TestMethod]
32+
[TestCategory("Sftp")]
33+
[TestCategory("integration")]
34+
[ExpectedException(typeof(SftpPermissionDeniedException))]
35+
public void Test_Sftp_EnumerateDirectory_Permission_Denied()
36+
{
37+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
38+
{
39+
sftp.Connect();
40+
41+
var files = sftp.EnumerateDirectory("/root");
42+
foreach (var file in files)
43+
{
44+
Debug.WriteLine(file.FullName);
45+
}
46+
47+
sftp.Disconnect();
48+
}
49+
}
50+
51+
[TestMethod]
52+
[TestCategory("Sftp")]
53+
[TestCategory("integration")]
54+
[ExpectedException(typeof(SftpPathNotFoundException))]
55+
public void Test_Sftp_EnumerateDirectory_Not_Exists()
56+
{
57+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
58+
{
59+
sftp.Connect();
60+
61+
var files = sftp.EnumerateDirectory("/asdfgh");
62+
foreach (var file in files)
63+
{
64+
Debug.WriteLine(file.FullName);
65+
}
66+
67+
sftp.Disconnect();
68+
}
69+
}
70+
71+
[TestMethod]
72+
[TestCategory("Sftp")]
73+
[TestCategory("integration")]
74+
public void Test_Sftp_EnumerateDirectory_Current()
75+
{
76+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
77+
{
78+
sftp.Connect();
79+
80+
var files = sftp.EnumerateDirectory(".");
81+
82+
Assert.IsTrue(files.Count() > 0);
83+
84+
foreach (var file in files)
85+
{
86+
Debug.WriteLine(file.FullName);
87+
}
88+
89+
sftp.Disconnect();
90+
}
91+
}
92+
93+
[TestMethod]
94+
[TestCategory("Sftp")]
95+
[TestCategory("integration")]
96+
public void Test_Sftp_EnumerateDirectory_Empty()
97+
{
98+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
99+
{
100+
sftp.Connect();
101+
102+
var files = sftp.EnumerateDirectory(string.Empty);
103+
104+
Assert.IsTrue(files.Count() > 0);
105+
106+
foreach (var file in files)
107+
{
108+
Debug.WriteLine(file.FullName);
109+
}
110+
111+
sftp.Disconnect();
112+
}
113+
}
114+
115+
[TestMethod]
116+
[TestCategory("Sftp")]
117+
[TestCategory("integration")]
118+
[Description("Test passing null to EnumerateDirectory.")]
119+
[ExpectedException(typeof(ArgumentNullException))]
120+
public void Test_Sftp_EnumerateDirectory_Null()
121+
{
122+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
123+
{
124+
sftp.Connect();
125+
126+
var files = sftp.EnumerateDirectory(null);
127+
128+
Assert.IsTrue(files.Count() > 0);
129+
130+
foreach (var file in files)
131+
{
132+
Debug.WriteLine(file.FullName);
133+
}
134+
135+
sftp.Disconnect();
136+
}
137+
}
138+
139+
[TestMethod]
140+
[TestCategory("Sftp")]
141+
[TestCategory("integration")]
142+
public void Test_Sftp_EnumerateDirectory_HugeDirectory()
143+
{
144+
var stopwatch = Stopwatch.StartNew();
145+
try
146+
{
147+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
148+
{
149+
sftp.Connect();
150+
sftp.ChangeDirectory("/home/" + Resources.USERNAME);
151+
152+
var count = 10000;
153+
// Create 10000 directory items
154+
for (int i = 0; i < count; i++)
155+
{
156+
sftp.CreateDirectory(string.Format("test_{0}", i));
157+
}
158+
Debug.WriteLine(string.Format("Created {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds));
159+
160+
stopwatch.Reset(); stopwatch.Start();
161+
var files = sftp.EnumerateDirectory(".");
162+
Debug.WriteLine(string.Format("Listed {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds));
163+
164+
// Ensure that directory has at least 10000 items
165+
stopwatch.Reset(); stopwatch.Start();
166+
var actualCount = files.Count();
167+
Assert.IsTrue(actualCount >= count);
168+
Debug.WriteLine(string.Format("Used {0} items within {1} seconds", actualCount, stopwatch.Elapsed.TotalSeconds));
169+
170+
sftp.Disconnect();
171+
}
172+
}
173+
finally
174+
{
175+
stopwatch.Reset(); stopwatch.Start();
176+
RemoveAllFiles();
177+
stopwatch.Stop();
178+
Debug.WriteLine(string.Format("Removed all files within {0} seconds", stopwatch.Elapsed.TotalSeconds));
179+
}
180+
}
181+
182+
[TestMethod]
183+
[TestCategory("Sftp")]
184+
[TestCategory("integration")]
185+
[ExpectedException(typeof(SshConnectionException))]
186+
public void Test_Sftp_EnumerateDirectory_After_Disconnected()
187+
{
188+
try {
189+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
190+
{
191+
sftp.Connect();
192+
193+
sftp.CreateDirectory("test_at_dsiposed");
194+
195+
var files = sftp.EnumerateDirectory(".").Take(1);
196+
197+
sftp.Disconnect();
198+
199+
// Must fail on disconnected session.
200+
var count = files.Count();
201+
}
202+
}
203+
finally
204+
{
205+
RemoveAllFiles();
206+
}
207+
}
208+
}
209+
}

Diff for: ‎src/Renci.SshNet.Tests/Classes/SftpClientTest.ListDirectory.cs

+32-16
Original file line numberDiff line numberDiff line change
@@ -140,26 +140,42 @@ public void Test_Sftp_ListDirectory_Null()
140140
[TestCategory("integration")]
141141
public void Test_Sftp_ListDirectory_HugeDirectory()
142142
{
143-
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
143+
var stopwatch = Stopwatch.StartNew();
144+
try
144145
{
145-
sftp.Connect();
146-
147-
// Create 10000 directory items
148-
for (int i = 0; i < 10000; i++)
146+
using (var sftp = new SftpClient(Resources.HOST, Resources.USERNAME, Resources.PASSWORD))
149147
{
150-
sftp.CreateDirectory(string.Format("test_{0}", i));
151-
Debug.WriteLine("Created " + i);
148+
sftp.Connect();
149+
sftp.ChangeDirectory("/home/" + Resources.USERNAME);
150+
151+
var count = 10000;
152+
// Create 10000 directory items
153+
for (int i = 0; i < count; i++)
154+
{
155+
sftp.CreateDirectory(string.Format("test_{0}", i));
156+
}
157+
Debug.WriteLine(string.Format("Created {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds));
158+
159+
stopwatch.Reset(); stopwatch.Start();
160+
var files = sftp.ListDirectory(".");
161+
Debug.WriteLine(string.Format("Listed {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds));
162+
163+
// Ensure that directory has at least 10000 items
164+
stopwatch.Reset(); stopwatch.Start();
165+
var actualCount = files.Count();
166+
Assert.IsTrue(actualCount >= count);
167+
Debug.WriteLine(string.Format("Used {0} items within {1} seconds", actualCount, stopwatch.Elapsed.TotalSeconds));
168+
169+
sftp.Disconnect();
152170
}
153-
154-
var files = sftp.ListDirectory(".");
155-
156-
// Ensure that directory has at least 10000 items
157-
Assert.IsTrue(files.Count() > 10000);
158-
159-
sftp.Disconnect();
160171
}
161-
162-
RemoveAllFiles();
172+
finally
173+
{
174+
stopwatch.Reset(); stopwatch.Start();
175+
RemoveAllFiles();
176+
stopwatch.Stop();
177+
Debug.WriteLine(string.Format("Removed all files within {0} seconds", stopwatch.Elapsed.TotalSeconds));
178+
}
163179
}
164180

165181
[TestMethod]

Diff for: ‎src/Renci.SshNet/ISftpClient.cs

+22
Original file line numberDiff line numberDiff line change
@@ -668,6 +668,28 @@ public interface ISftpClient
668668
/// <exception cref="ObjectDisposedException">The method was called after the client was disposed.</exception>
669669
IEnumerable<SftpFile> ListDirectory(string path, Action<int> listCallback = null);
670670

671+
/// <summary>
672+
/// Enumerates files and directories in remote directory.
673+
/// </summary>
674+
/// <remarks>
675+
/// This method differs to <see cref="ListDirectory(string, Action{int})"/> in the way how the items are returned.
676+
/// It yields the items to the last moment for the enumerator to decide if it needs to continue or stop enumerating the items.
677+
/// It is handy in case of really huge directory contents at remote server - meaning really huge 65 thousand files and more.
678+
/// It also decrease the memory footprint and avoids LOH allocation as happen per call to <see cref="ListDirectory(string, Action{int})"/> method.
679+
/// There aren't asynchronous counterpart methods to this because enumerating should happen in your specific asynchronous block.
680+
/// </remarks>
681+
/// <param name="path">The path.</param>
682+
/// <param name="listCallback">The list callback.</param>
683+
/// <returns>
684+
/// An <see cref="System.Collections.Generic.IEnumerable{SftpFile}"/> of files and directories ready to be enumerated.
685+
/// </returns>
686+
/// <exception cref="ArgumentNullException"><paramref name="path" /> is <b>null</b>.</exception>
687+
/// <exception cref="SshConnectionException">Client is not connected.</exception>
688+
/// <exception cref="SftpPermissionDeniedException">Permission to list the contents of the directory was denied by the remote host. <para>-or-</para> A SSH command was denied by the server.</exception>
689+
/// <exception cref="SshException">A SSH error where <see cref="Exception.Message" /> is the message from the remote host.</exception>
690+
/// <exception cref="ObjectDisposedException">The method was called after the client was disposed.</exception>
691+
IEnumerable<SftpFile> EnumerateDirectory(string path, Action<int> listCallback = null);
692+
671693
/// <summary>
672694
/// Opens a <see cref="SftpFileStream"/> on the specified path with read/write access.
673695
/// </summary>

0 commit comments

Comments
 (0)