Web · Wiki · Activities · Blog · Lists · Chat · Meeting · Bugs · Git · Translate · Archive · People · Donate

Commit a08ad66de6fc06e003b58b5cdc00c6fe3fca4270

add preliminary support for passing data by fd

Passing data out-of-band using data paths has a couple of issues (different
user, chroot, ownership / deletion, in-place modification by callers, etc.)
that passing data via file descriptors can avoid. Recent versions of D-Bus and
python-dbus have support for fd passing, so we can add alternative versions of
the functions that take a data path and have the alternatives take a file
descriptor instead.

Support for this is still suboptimal as we make a copy and pass a path
internally. We can easily improve on that later as it's just a matter of
performance, not functionality.

We may even consider dropping the non-fd counterparts. However, since D-Bus
still does not support a "maybe" type, we'll need alternative versions of these
calls that don't take a file descriptor, for use with metadata-only entries. We
could also pass a reference to /dev/null, but that's more a hack than a
solution.
  
125125 async_err_cb=async_err_cb)
126126
127127 @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
128 in_signature='a{sv}h', out_signature='ss',
129 async_callbacks=('async_cb', 'async_err_cb'),
130 byte_arrays=True)
131 def create_fd(self, metadata, data_fd, async_cb, async_err_cb):
132 """
133 - add new entry, assign ids
134 """
135 # FIXME: avoid unnecessary copy, instead change
136 # InternalAPI.save() to take fd and adapt existing callers.
137 tmp_file = tempfile.NamedTemporaryFile(delete=False)
138 with os.fdopen(data_fd.take(), 'rb') as data_file:
139 shutil.copyfileobj(data_file, tmp_file)
140
141 tmp_file.flush()
142 self._internal_api.save(tree_id='', parent_id='', metadata=metadata,
143 path=tmp_file.name, delete_after=True,
144 async_cb=async_cb,
145 async_err_cb=async_err_cb)
146
147 @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
128148 in_signature='ssa{sv}s', out_signature='s',
129149 async_callbacks=('async_cb', 'async_err_cb'),
130150 byte_arrays=True)
168168 async_err_cb=async_err_cb)
169169
170170 @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
171 in_signature='ssa{sv}h', out_signature='s',
172 async_callbacks=('async_cb', 'async_err_cb'),
173 byte_arrays=True)
174 def add_version_fd(self, tree_id, parent_id, metadata, data_fd, async_cb,
175 async_err_cb):
176 """
177 - add new version to existing object
178 """
179 def success_cb(tree_id, child_id):
180 async_cb(child_id)
181
182 if not tree_id:
183 raise ValueError('No tree_id given')
184
185 if not parent_id:
186 raise ValueError('No parent_id given')
187
188 # FIXME: avoid unnecessary copy, instead change
189 # InternalAPI.save() to take fd and adapt existing callers.
190 tmp_file = tempfile.NamedTemporaryFile(delete=False)
191 with os.fdopen(data_fd.take(), 'rb') as data_file:
192 shutil.copyfileobj(data_file, tmp_file)
193
194 tmp_file.flush()
195 self._internal_api.save(tree_id, parent_id, metadata,
196 path=tmp_file.name, delete_after=True,
197 async_cb=success_cb,
198 async_err_cb=async_err_cb)
199
200 @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
171201 in_signature='ssa{sv}', out_signature='',
172202 byte_arrays=True)
173203 def change_metadata(self, tree_id, version_id, metadata):
227227 return self._internal_api.get_data_path(object_id, sender=sender)
228228
229229 @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
230 in_signature='ss', out_signature='h',
231 sender_keyword='sender')
232 def get_data_fd(self, tree_id, version_id, sender=None):
233 object_id = (tree_id, version_id)
234 path = self._internal_api.get_data_path(object_id, sender=sender)
235 return os.open(path, os.O_RDONLY)
236
237 @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
230238 in_signature='ss', out_signature='a{sv}')
231239 def get_metadata(self, tree_id, version_id):
232240 object_id = (tree_id, version_id)
261261
262262 metadata['version_id'] = version_id
263263 self._internal_api.save(tree_id, parent_id, metadata, data_path,
264 delete_after=True, allow_new_parent=True,
265 async_cb=async_cb,
266 async_err_cb=async_err_cb)
267
268 @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
269 in_signature='sssa{sv}s', out_signature='ss',
270 async_callbacks=('async_cb', 'async_err_cb'),
271 byte_arrays=True)
272 def restore_fd(self, tree_id, parent_id, version_id, metadata, data_fd,
273 async_cb, async_err_cb):
274 """
275 - add a new version with the given ids
276 - there must be no existing entry with the same (tree_id, version_id)
277 """
278 if not tree_id:
279 raise ValueError('No tree_id given')
280
281 metadata['version_id'] = version_id
282 # FIXME: avoid unnecessary copy, instead change
283 # InternalAPI.save() to take fd and adapt existing callers.
284 tmp_file = tempfile.NamedTemporaryFile(delete=False)
285 with os.fdopen(data_fd.take(), 'rb') as data_file:
286 shutil.copyfileobj(data_file, tmp_file)
287
288 tmp_file.flush()
289 self._internal_api.save(tree_id, parent_id, metadata, tmp_file.name,
264290 delete_after=True, allow_new_parent=True,
265291 async_cb=async_cb,
266292 async_err_cb=async_err_cb)
  
119119>>> dog_oid = ds.create(dog_props, dog_file_name)
120120
121121
122You can also pass the data as a file descriptor, useful for streaming:
123>>> dog2_file = tempfile.TemporaryFile()
124>>> dog2_file.write(dog_content)
125>>> dog2_file.flush()
126>>> dog2_file.seek(0)
127>>> dog2_oid = ds.create_fd({'title': 'DS test object 4', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest1'}, dog2_file.fileno())
128>>> dog2_file.close()
129
130
122131Retrieve and verify the entry with content:
123132>>> dog_retrieved = ds.get_data_path(*dog_oid)
124133>>> assert(file(dog_retrieved).read() == dog_content)
125134>>> os.remove(dog_retrieved)
126135
136The same using file descriptor passing:
137>>> dog2_retrieved_file = os.fdopen(ds.get_data_fd(*dog2_oid).take(), 'r')
138>>> assert(dog2_retrieved_file.read() == dog_content)
127139
140
128141Update the entry content (creates a new version):
129142>>> dog_content_2 = 'The quick brown fox jumped over the lazy dog.'
130143>>> dog_file_name = create_temp_file(dog_content_2)
131144>>> dog_updated_version_id = ds.add_version(dog_oid[0], dog_oid[1], dog_props, dog_file_name)
145
146
147Update the entry content by passing a file descriptor:
148>>> dog_content_2 = 'The quick brown fox jumped over the lazy dog.'
149>>> dog2_file = tempfile.TemporaryFile()
150>>> dog2_file.write(dog_content)
151>>> dog2_file.flush()
152>>> dog2_file.seek(0)
153>>> dog2_updated_version_id = ds.add_version_fd(dog_oid[0], dog_oid[1], dog_props, dog2_file.fileno())
132154
133155
134156Verify updated content: