Skip to content

Commit d02b996

Browse files
author
Omar Marzouk
committed
Linting
1 parent 8e3a32f commit d02b996

File tree

4 files changed

+143
-30
lines changed

4 files changed

+143
-30
lines changed

Diff for: ‎docs/tutorials/daos.ipynb

+122-8
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,31 @@
11
{
22
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {},
6+
"source": [
7+
"##### Copyright 2022 The TensorFlow IO Authors."
8+
]
9+
},
10+
{
11+
"cell_type": "code",
12+
"execution_count": null,
13+
"metadata": {},
14+
"outputs": [],
15+
"source": [
16+
"#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
17+
"# you may not use this file except in compliance with the License.\n",
18+
"# You may obtain a copy of the License at\n",
19+
"#\n",
20+
"# https://www.apache.org/licenses/LICENSE-2.0\n",
21+
"#\n",
22+
"# Unless required by applicable law or agreed to in writing, software\n",
23+
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
24+
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
25+
"# See the License for the specific language governing permissions and\n",
26+
"# limitations under the License."
27+
]
28+
},
329
{
430
"cell_type": "markdown",
531
"id": "1f9e30da",
@@ -10,6 +36,26 @@
1036
"# DAOS Filesystem with Tensorflow (Using MNIST)"
1137
]
1238
},
39+
{
40+
"cell_type": "markdown",
41+
"metadata": {},
42+
"source": [
43+
"<table class=\"tfo-notebook-buttons\" align=\"left\">\n",
44+
" <td>\n",
45+
" <a target=\"_blank\" href=\"https://www.tensorflow.org/io/tutorials/daos\"><img src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" />View on TensorFlow.org</a>\n",
46+
" </td>\n",
47+
" <td>\n",
48+
" <a target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/io/blob/master/docs/tutorials/daos.ipynb\"><img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>\n",
49+
" </td>\n",
50+
" <td>\n",
51+
" <a target=\"_blank\" href=\"https://github.com/tensorflow/io/blob/master/docs/tutorials/daos.ipynb\"><img src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a>\n",
52+
" </td>\n",
53+
" <td>\n",
54+
" <a href=\"https://storage.googleapis.com/tensorflow_docs/io/docs/tutorials/daos.ipynb\"><img src=\"https://www.tensorflow.org/images/download_logo_32px.png\" />Download notebook</a>\n",
55+
" </td>\n",
56+
"</table>"
57+
]
58+
},
1359
{
1460
"cell_type": "markdown",
1561
"id": "22b37505",
@@ -21,7 +67,7 @@
2167
"\n",
2268
"This tutorial shows how to use read and write files on [DAOS Filesystem](https://docs.daos.io/) with TensorFlow, through TensorFlow IO's DAOS file system integration.\n",
2369
"\n",
24-
"A machine running DAOS natively or through a [docker emulator](https://github.com/daos-stack/daos/tree/master/utils/docker) is needed to run this tutorial and/or use the Tensorflow IO DAOS integration. The DAOS Pool and Container used for this tutorial will be created and deleted within this tutorial, where we will be training and testing a simple Neural Network on the MNIST Dataset loaded from the DAOS File System Plugin.\n",
70+
"A machine running DAOS natively or through a [docker emulator](https://github.com/daos-stack/daos/tree/master/utils/docker) is needed to run this tutorial and/or use the Tensorflow IO DAOS integration. The DAOS Pool and Container used for this tutorial will be created and deleted within this tutorial, where you will be training and testing a simple Neural Network on the MNIST Dataset loaded from the DAOS File System Plugin.\n",
2571
"\n",
2672
"The pool and container id or label are part of the filename uri:\n",
2773
"```\n",
@@ -52,12 +98,59 @@
5298
},
5399
{
54100
"cell_type": "code",
55-
"execution_count": null,
101+
"execution_count": 1,
56102
"id": "5e35916b",
57103
"metadata": {
58104
"id": "5de1951509cb"
59105
},
60-
"outputs": [],
106+
"outputs": [
107+
{
108+
"name": "stdout",
109+
"output_type": "stream",
110+
"text": [
111+
"Defaulting to user installation because normal site-packages is not writeable\n",
112+
"Requirement already satisfied: tensorflow-io in /home/omar/.local/lib/python3.8/site-packages (0.20.0)\n",
113+
"Requirement already satisfied: tensorflow<2.7.0,>=2.6.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow-io) (2.6.0)\n",
114+
"Requirement already satisfied: tensorflow-io-gcs-filesystem==0.20.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow-io) (0.20.0)\n",
115+
"Requirement already satisfied: gast==0.4.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (0.4.0)\n",
116+
"Requirement already satisfied: grpcio<2.0,>=1.37.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.39.0)\n",
117+
"Requirement already satisfied: protobuf>=3.9.2 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (3.17.3)\n",
118+
"Requirement already satisfied: tensorboard~=2.6 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (2.6.0)\n",
119+
"Requirement already satisfied: tensorflow-estimator~=2.6 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (2.6.0)\n",
120+
"Requirement already satisfied: typing-extensions~=3.7.4 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (3.7.4.3)\n",
121+
"Requirement already satisfied: termcolor~=1.1.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.1.0)\n",
122+
"Requirement already satisfied: wrapt~=1.12.1 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.12.1)\n",
123+
"Requirement already satisfied: google-pasta~=0.2 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (0.2.0)\n",
124+
"Requirement already satisfied: keras~=2.6 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (2.6.0)\n",
125+
"Requirement already satisfied: six~=1.15.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.15.0)\n",
126+
"Requirement already satisfied: numpy~=1.19.2 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.19.5)\n",
127+
"Requirement already satisfied: opt-einsum~=3.3.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (3.3.0)\n",
128+
"Requirement already satisfied: wheel~=0.35 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (0.37.0)\n",
129+
"Requirement already satisfied: astunparse~=1.6.3 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.6.3)\n",
130+
"Requirement already satisfied: absl-py~=0.10 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (0.13.0)\n",
131+
"Requirement already satisfied: keras-preprocessing~=1.1.2 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.1.2)\n",
132+
"Requirement already satisfied: h5py~=3.1.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (3.1.0)\n",
133+
"Requirement already satisfied: clang~=5.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (5.0)\n",
134+
"Requirement already satisfied: flatbuffers~=1.12.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.12)\n",
135+
"Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /home/omar/.local/lib/python3.8/site-packages (from tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (0.4.6)\n",
136+
"Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.8.0)\n",
137+
"Requirement already satisfied: markdown>=2.6.8 in /home/omar/.local/lib/python3.8/site-packages (from tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (3.3.4)\n",
138+
"Requirement already satisfied: google-auth<2,>=1.6.3 in /home/omar/.local/lib/python3.8/site-packages (from tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.35.0)\n",
139+
"Requirement already satisfied: setuptools>=41.0.0 in /usr/lib/python3/dist-packages (from tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (45.2.0)\n",
140+
"Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /home/omar/.local/lib/python3.8/site-packages (from tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (0.6.1)\n",
141+
"Requirement already satisfied: requests<3,>=2.21.0 in /usr/lib/python3/dist-packages (from tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (2.22.0)\n",
142+
"Requirement already satisfied: werkzeug>=0.11.15 in /home/omar/.local/lib/python3.8/site-packages (from tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (2.0.1)\n",
143+
"Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/omar/.local/lib/python3.8/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (0.2.8)\n",
144+
"Requirement already satisfied: rsa<5,>=3.1.4 in /home/omar/.local/lib/python3.8/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (4.7.2)\n",
145+
"Requirement already satisfied: cachetools<5.0,>=2.0.0 in /home/omar/.local/lib/python3.8/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (4.2.2)\n",
146+
"Requirement already satisfied: requests-oauthlib>=0.7.0 in /home/omar/.local/lib/python3.8/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (1.3.0)\n",
147+
"Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /home/omar/.local/lib/python3.8/site-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (0.4.8)\n",
148+
"Requirement already satisfied: oauthlib>=3.0.0 in /usr/lib/python3/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow<2.7.0,>=2.6.0->tensorflow-io) (3.1.0)\n",
149+
"\u001b[33mWARNING: You are using pip version 21.2.4; however, version 22.0.4 is available.\n",
150+
"You should consider upgrading via the '/usr/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n"
151+
]
152+
}
153+
],
61154
"source": [
62155
"try:\n",
63156
" %tensorflow_version 2.x \n",
@@ -79,12 +172,21 @@
79172
},
80173
{
81174
"cell_type": "code",
82-
"execution_count": null,
175+
"execution_count": 1,
83176
"id": "79528fed",
84177
"metadata": {
85178
"id": "fb83b02da201"
86179
},
87-
"outputs": [],
180+
"outputs": [
181+
{
182+
"name": "stdout",
183+
"output_type": "stream",
184+
"text": [
185+
"/usr/bin/sh: 1: dmg: not found\n",
186+
"/usr/bin/sh: 1: daos: not found\n"
187+
]
188+
}
189+
],
88190
"source": [
89191
"!dmg -i pool create -s 500M TEST_POOL\n",
90192
"!daos cont create --pool=TEST_POOL --type=POSIX TEST_CONT"
@@ -128,7 +230,7 @@
128230
},
129231
"outputs": [],
130232
"source": [
131-
"dfs_url = \"dfs://TEST_POOL/TEST_CONT/\" # This the path we'll be using to load and access the dataset\n",
233+
"dfs_url = \"dfs://TEST_POOL/TEST_CONT/\" # This the path you'll be using to load and access the dataset\n",
132234
"pwd = !pwd\n",
133235
"posix_url = pwd[0] + \"/tests/test_dfs/\""
134236
]
@@ -162,7 +264,7 @@
162264
"id": "b40e9a30808c"
163265
},
164266
"source": [
165-
"Copying the Data from the POSIX Filesystem to the DAOS Filesystem under the pool and container we just created"
267+
"Copying the Data from the POSIX Filesystem to the DAOS Filesystem under the pool and container you just created"
166268
]
167269
},
168270
{
@@ -270,7 +372,7 @@
270372
"id": "4129e8e2c1b4"
271373
},
272374
"source": [
273-
"Compiling the model we just built"
375+
"Compiling the model you just built"
274376
]
275377
},
276378
{
@@ -467,6 +569,18 @@
467569
"kernelspec": {
468570
"display_name": "Python 3",
469571
"name": "python3"
572+
},
573+
"language_info": {
574+
"codemirror_mode": {
575+
"name": "ipython",
576+
"version": 3
577+
},
578+
"file_extension": ".py",
579+
"mimetype": "text/x-python",
580+
"name": "python",
581+
"nbconvert_exporter": "python",
582+
"pygments_lexer": "ipython3",
583+
"version": "3.8.10"
470584
}
471585
},
472586
"nbformat": 4,

Diff for: ‎tensorflow_io/core/filesystems/dfs/dfs_filesystem.cc

+14-17
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,20 @@
1-
#include "tensorflow_io/core/filesystems/dfs/dfs_utils.h"
2-
31
#include <stdio.h>
2+
3+
#include "tensorflow_io/core/filesystems/dfs/dfs_utils.h"
44
#undef NDEBUG
55
#include <cassert>
66

77
namespace tensorflow {
88
namespace io {
99
namespace dfs {
1010

11-
1211
// SECTION 1. Implementation for `TF_RandomAccessFile`
1312
// ----------------------------------------------------------------------------
1413
namespace tf_random_access_file {
1514
typedef struct DFSRandomAccessFile {
1615
std::string dfs_path;
1716
dfs_t* daos_fs;
18-
dfs_obj_t *daos_file;
17+
dfs_obj_t* daos_file;
1918
std::vector<ReadBuffer> buffers;
2019
daos_size_t file_size;
2120
daos_handle_t mEventQueueHandle{};
@@ -77,7 +76,7 @@ int64_t Read(const TF_RandomAccessFile* file, uint64_t offset, size_t n,
7776
if (read_buf.CacheHit(curr_offset)) {
7877
read_bytes = read_buf.CopyFromCache(ret, ret_offset, curr_offset, n,
7978
dfs_file->file_size, status);
80-
break;
79+
break;
8180
}
8281
}
8382

@@ -96,8 +95,8 @@ int64_t Read(const TF_RandomAccessFile* file, uint64_t offset, size_t n,
9695
size_t async_offset = curr_offset;
9796
for (size_t i = 0; i < dfs_file->buffers.size(); i++) {
9897
if (async_offset > dfs_file->file_size) break;
99-
dfs_file->buffers[i].ReadAsync(dfs_file->daos_fs,
100-
dfs_file->daos_file, async_offset);
98+
dfs_file->buffers[i].ReadAsync(dfs_file->daos_fs, dfs_file->daos_file,
99+
async_offset);
101100
async_offset += BUFF_SIZE;
102101
}
103102
}
@@ -113,18 +112,18 @@ namespace tf_writable_file {
113112
typedef struct DFSWritableFile {
114113
std::string dfs_path;
115114
dfs_t* daos_fs;
116-
dfs_obj_t *daos_file;
115+
dfs_obj_t* daos_file;
117116
daos_size_t file_size;
118117
bool size_known;
119118

120119
DFSWritableFile(std::string dfs_path, dfs_t* file_system, dfs_obj_t* obj)
121120
: dfs_path(std::move(dfs_path)) {
122121
daos_fs = file_system;
123122
daos_file = obj;
124-
size_known=false;
123+
size_known = false;
125124
}
126125

127-
int get_file_size(daos_size_t &size) {
126+
int get_file_size(daos_size_t& size) {
128127
if (!size_known) {
129128
int rc = dfs_get_size(daos_fs, daos_file, &file_size);
130129
if (rc != 0) {
@@ -141,9 +140,7 @@ typedef struct DFSWritableFile {
141140
size_known = true;
142141
}
143142

144-
void unset_file_size(void) {
145-
size_known = false;
146-
}
143+
void unset_file_size(void) { size_known = false; }
147144
} DFSWritableFile;
148145

149146
void Cleanup(TF_WritableFile* file) {
@@ -171,8 +168,8 @@ void Append(const TF_WritableFile* file, const char* buffer, size_t n,
171168
return;
172169
}
173170

174-
rc = dfs_write(dfs_file->daos_fs, dfs_file->daos_file, &wsgl,
175-
cur_file_size, NULL);
171+
rc = dfs_write(dfs_file->daos_fs, dfs_file->daos_file, &wsgl, cur_file_size,
172+
NULL);
176173
if (rc) {
177174
TF_SetStatus(status, TF_RESOURCE_EXHAUSTED, "");
178175
dfs_file->unset_file_size();
@@ -274,8 +271,8 @@ void NewRandomAccessFile(const TF_Filesystem* filesystem, const char* path,
274271
}
275272
auto random_access_file =
276273
new tf_random_access_file::DFSRandomAccessFile(path, daos->daos_fs, obj);
277-
random_access_file->buffers[0].ReadAsync(
278-
daos->daos_fs, random_access_file->daos_file, 0);
274+
random_access_file->buffers[0].ReadAsync(daos->daos_fs,
275+
random_access_file->daos_file, 0);
279276
file->plugin_file = random_access_file;
280277
TF_SetStatus(status, TF_OK, "");
281278
}

Diff for: ‎tensorflow_io/core/filesystems/dfs/dfs_utils.cc

+5-3
Original file line numberDiff line numberDiff line change
@@ -556,7 +556,8 @@ int ReadBuffer::ReadAsync(dfs_t* daos_fs, dfs_obj_t* file, const size_t off) {
556556
assert(rc == 0);
557557
rc = daos_event_init(event, eqh, nullptr);
558558
assert(rc == 0);
559-
event->ev_error = dfs_read(daos_fs, file, &rsgl, buffer_offset, &read_size, event);
559+
event->ev_error =
560+
dfs_read(daos_fs, file, &rsgl, buffer_offset, &read_size, event);
560561
return 0;
561562
}
562563

@@ -571,8 +572,9 @@ int ReadBuffer::CopyData(char* ret, const size_t ret_offset, const size_t off,
571572
}
572573

573574
int64_t ReadBuffer::CopyFromCache(char* ret, const size_t ret_offset,
574-
const size_t off, const size_t n,
575-
const daos_size_t file_size, TF_Status* status) {
575+
const size_t off, const size_t n,
576+
const daos_size_t file_size,
577+
TF_Status* status) {
576578
size_t read_size;
577579
read_size = off + n > file_size ? file_size - off : n;
578580
read_size = off + read_size > buffer_offset + buffer_size

Diff for: ‎tensorflow_io/core/filesystems/dfs/dfs_utils.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -208,8 +208,8 @@ class ReadBuffer {
208208
const size_t n);
209209

210210
int64_t CopyFromCache(char* ret, const size_t ret_offset, const size_t off,
211-
const size_t n, const daos_size_t file_size,
212-
TF_Status* status);
211+
const size_t n, const daos_size_t file_size,
212+
TF_Status* status);
213213

214214
private:
215215
size_t id;

0 commit comments

Comments
 (0)