@@ -104,7 +104,7 @@ def write(self, df, path, compression='snappy',
104
104
coerce_timestamps = 'ms' , index = None , partition_cols = None ,
105
105
** kwargs ):
106
106
self .validate_dataframe (df )
107
- path , _ , _ , _ = get_filepath_or_buffer (path , mode = 'wb' )
107
+ path , _ , _ , should_close = get_filepath_or_buffer (path , mode = 'wb' )
108
108
109
109
if index is None :
110
110
from_pandas_kwargs = {}
@@ -121,6 +121,13 @@ def write(self, df, path, compression='snappy',
121
121
table , path , compression = compression ,
122
122
coerce_timestamps = coerce_timestamps , ** kwargs )
123
123
124
+ if should_close :
125
+ try :
126
+ path .close ()
127
+ except AttributeError :
128
+ # path is not file-like (e.g. a string)
129
+ pass
130
+
124
131
def read (self , path , columns = None , ** kwargs ):
125
132
path , _ , _ , should_close = get_filepath_or_buffer (path )
126
133
@@ -130,7 +137,8 @@ def read(self, path, columns=None, **kwargs):
130
137
if should_close :
131
138
try :
132
139
path .close ()
133
- except : # noqa: flake8
140
+ except AttributeError :
141
+ # path is not file-like (e.g. a string)
134
142
pass
135
143
136
144
return result
@@ -183,17 +191,24 @@ def write(self, df, path, compression='snappy', index=None,
183
191
# path is s3:// so we need to open the s3file in 'wb' mode.
184
192
# TODO: Support 'ab'
185
193
186
- path , _ , _ , _ = get_filepath_or_buffer (path , mode = 'wb' )
194
+ path , _ , _ , should_close = get_filepath_or_buffer (path , mode = 'wb' )
187
195
# And pass the opened s3file to the fastparquet internal impl.
188
196
kwargs ['open_with' ] = lambda path , _ : path
189
197
else :
190
- path , _ , _ , _ = get_filepath_or_buffer (path )
198
+ path , _ , _ , should_close = get_filepath_or_buffer (path )
191
199
192
200
with catch_warnings (record = True ):
193
201
self .api .write (path , df , compression = compression ,
194
202
write_index = index , partition_on = partition_cols ,
195
203
** kwargs )
196
204
205
+ if should_close :
206
+ try :
207
+ path .close ()
208
+ except AttributeError :
209
+ # path is not file-like (e.g. a string)
210
+ pass
211
+
197
212
def read (self , path , columns = None , ** kwargs ):
198
213
if is_s3_url (path ):
199
214
# When path is s3:// an S3File is returned.
@@ -205,9 +220,16 @@ def read(self, path, columns=None, **kwargs):
205
220
finally :
206
221
s3 .close ()
207
222
else :
208
- path , _ , _ , _ = get_filepath_or_buffer (path )
223
+ path , _ , _ , should_close = get_filepath_or_buffer (path )
209
224
parquet_file = self .api .ParquetFile (path )
210
225
226
+ if should_close :
227
+ try :
228
+ path .close ()
229
+ except (AttributeError , OSError ):
230
+ # path is not file-like (e.g. a string)
231
+ pass
232
+
211
233
return parquet_file .to_pandas (columns = columns , ** kwargs )
212
234
213
235
0 commit comments