Skip to content

Commit 060a985

Browse files
committed
fs: copyfile: don't touch callback at all for <1G files
This again wastes a lot of time, as `set_size` might trigger a pbar refresh. Related #229 Related iterative/dvc#9914
1 parent 4a68565 commit 060a985

File tree

1 file changed

+40
-26
lines changed

1 file changed

+40
-26
lines changed

src/dvc_objects/fs/utils.py

+40-26
Original file line numberDiff line numberDiff line change
@@ -142,48 +142,62 @@ def makedirs(path, exist_ok: bool = False, mode: Optional[int] = None) -> None:
142142
)
143143

144144

145-
def copyfile(
145+
def _copyfile_with_pbar(
146146
src: "AnyFSPath",
147147
dest: "AnyFSPath",
148+
total: int,
148149
callback: Optional["Callback"] = None,
149150
no_progress_bar: bool = False,
150151
name: Optional[str] = None,
152+
):
153+
from .callbacks import Callback
154+
155+
name = name if name else os.path.basename(dest)
156+
157+
if callback:
158+
callback.set_size(total)
159+
160+
with open(src, "rb") as fsrc, open(dest, "wb+") as fdest:
161+
with Callback.as_tqdm_callback(
162+
callback,
163+
size=total,
164+
bytes=True,
165+
disable=no_progress_bar,
166+
desc=name,
167+
) as cb:
168+
wrapped = cb.wrap_attr(fdest, "write")
169+
while True:
170+
buf = fsrc.read(LOCAL_CHUNK_SIZE)
171+
if not buf:
172+
break
173+
wrapped.write(buf)
174+
175+
if callback:
176+
callback.absolute_update(total)
177+
178+
179+
def copyfile(
180+
src: "AnyFSPath",
181+
dest: "AnyFSPath",
182+
**kwargs,
151183
) -> None:
152184
"""Copy file with progress bar"""
153-
name = name if name else os.path.basename(dest)
154185
total = os.stat(src).st_size
155186

156187
if os.path.isdir(dest):
157188
dest = os.path.join(dest, os.path.basename(src))
158189

159-
if callback:
160-
callback.set_size(total)
161-
162190
try:
163191
system.reflink(src, dest)
192+
return
164193
except OSError:
165-
if total < COPY_PBAR_MIN_SIZE:
166-
shutil.copyfile(src, dest)
167-
else:
168-
from .callbacks import Callback
169-
170-
with open(src, "rb") as fsrc, open(dest, "wb+") as fdest:
171-
with Callback.as_tqdm_callback(
172-
callback,
173-
size=total,
174-
bytes=True,
175-
disable=no_progress_bar,
176-
desc=name,
177-
) as cb:
178-
wrapped = cb.wrap_attr(fdest, "write")
179-
while True:
180-
buf = fsrc.read(LOCAL_CHUNK_SIZE)
181-
if not buf:
182-
break
183-
wrapped.write(buf)
194+
pass
184195

185-
if callback:
186-
callback.absolute_update(total)
196+
if total < COPY_PBAR_MIN_SIZE:
197+
shutil.copyfile(src, dest)
198+
return
199+
200+
_copyfile_with_pbar(src, dest, total, **kwargs)
187201

188202

189203
def tmp_fname(fname: "AnyFSPath" = "") -> "AnyFSPath":

0 commit comments

Comments
 (0)