Web · Wiki · Activities · Blog · Lists · Chat · Meeting · Bugs · Git · Translate · Archive · People · Donate
1
#
2
# Author: Sascha Silbe <sascha-pgp@silbe.org>
3
#
4
# This program is free software; you can redistribute it and/or modify
5
# it under the terms of the GNU General Public License version 3
6
# as published by the Free Software Foundation.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program. If not, see <http://www.gnu.org/licenses/>.
15
"""
16
Gdatastore D-Bus service API
17
"""
18
19
import hashlib
20
import logging
21
import os
22
import pprint
23
import shutil
24
from subprocess import Popen, PIPE
25
import tempfile
26
import time
27
import uuid
28
29
import dbus
30
import dbus.service
31
import gconf
32
33
from gdatastore.index import Index
34
35
36
DBUS_SERVICE_NATIVE_V1 = 'org.silbe.GDataStore'
37
DBUS_INTERFACE_NATIVE_V1 = 'org.silbe.GDataStore1'
38
DBUS_PATH_NATIVE_V1 = '/org/silbe/GDataStore1'
39
40
DBUS_SERVICE_SUGAR_V2 = 'org.laptop.sugar.DataStore'
41
DBUS_INTERFACE_SUGAR_V2 = 'org.laptop.sugar.DataStore'
42
DBUS_PATH_SUGAR_V2 = '/org/laptop/sugar/DataStore'
43
44
DBUS_SERVICE_SUGAR_V3 = 'org.laptop.sugar.DataStore'
45
DBUS_INTERFACE_SUGAR_V3 = 'org.laptop.sugar.DataStore2'
46
DBUS_PATH_SUGAR_V3 = '/org/laptop/sugar/DataStore2'
47
48
49
class DataStoreError(Exception):
50
    pass
51
52
53
class GitError(DataStoreError):
54
    def __init__(self, returncode, stderr):
55
        self.returncode = returncode
56
        self.stderr = unicode(stderr)
57
        Exception.__init__(self)
58
59
    def __unicode__(self):
60
        return u'Git returned with exit code #%d: %s' % (self.returncode,
61
                                                         self.stderr)
62
63
    def __str__(self):
64
        return self.__unicode__()
65
66
67
class DBusApiNativeV1(dbus.service.Object):
68
    """Native gdatastore D-Bus API
69
    """
70
71
    def __init__(self, internal_api):
72
        self._internal_api = internal_api
73
        bus_name = dbus.service.BusName(DBUS_SERVICE_NATIVE_V1,
74
                                        bus=dbus.SessionBus(),
75
                                        replace_existing=False,
76
                                        allow_replacement=False,
77
                                        do_not_queue=True)
78
        dbus.service.Object.__init__(self, bus_name, DBUS_PATH_NATIVE_V1)
79
        self._internal_api.add_callback('change_metadata',
80
                                        self.__change_metadata_cb)
81
        self._internal_api.add_callback('delete', self.__delete_cb)
82
        self._internal_api.add_callback('save', self.__save_cb)
83
84
    @dbus.service.signal(DBUS_INTERFACE_NATIVE_V1, signature='sssa{sv}')
85
    def AddedNewVersion(self, tree_id, child_id, parent_id, metadata):
86
        # pylint: disable-msg=C0103
87
        pass
88
89
    @dbus.service.signal(DBUS_INTERFACE_NATIVE_V1, signature='ssa{sv}')
90
    def Created(self, tree_id, child_id, metadata):
91
        # pylint: disable-msg=C0103
92
        pass
93
94
    @dbus.service.signal(DBUS_INTERFACE_NATIVE_V1, signature='ssa{sv}')
95
    def ChangedMetadata(self, tree_id, version_id, metadata):
96
        # pylint: disable-msg=C0103
97
        pass
98
99
    @dbus.service.signal(DBUS_INTERFACE_NATIVE_V1, signature='ss')
100
    def Deleted(self, tree_id, version_id):
101
        # pylint: disable-msg=C0103
102
        pass
103
104
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
105
                         in_signature='a{sv}s', out_signature='ss',
106
                         async_callbacks=('async_cb', 'async_err_cb'),
107
                         byte_arrays=True)
108
    def create(self, metadata, data_path, async_cb, async_err_cb):
109
        """
110
        - add new entry, assign ids
111
        - data='' indicates no data to store
112
        - bad design? (data OOB)
113
        """
114
        # TODO: what about transfer_ownership/delete_after?
115
        self._internal_api.save(tree_id='', parent_id='', metadata=metadata,
116
                                path=data_path, delete_after=True,
117
                                async_cb=async_cb,
118
                                async_err_cb=async_err_cb)
119
120
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
121
                         in_signature='ssa{sv}s', out_signature='s',
122
                         async_callbacks=('async_cb', 'async_err_cb'),
123
                         byte_arrays=True)
124
    def add_version(self, tree_id, parent_id, metadata, data_path, async_cb,
125
                    async_err_cb):
126
        """
127
        - add new version to existing object
128
        """
129
        def success_cb(tree_id, child_id):
130
            async_cb(child_id)
131
132
        if not tree_id:
133
            raise ValueError('No tree_id given')
134
135
        if not parent_id:
136
            raise ValueError('No parent_id given')
137
138
        self._internal_api.save(tree_id, parent_id, metadata, data_path,
139
                                delete_after=True,
140
                                async_cb=success_cb,
141
                                async_err_cb=async_err_cb)
142
143
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
144
                         in_signature='ssa{sv}', out_signature='',
145
                         byte_arrays=True)
146
    def change_metadata(self, tree_id, version_id, metadata):
147
        """
148
        - change the metadata of an existing version
149
        """
150
        object_id = (tree_id, version_id)
151
        self._internal_api.change_metadata(object_id, metadata)
152
153
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
154
                         in_signature='ss', out_signature='')
155
    def delete(self, tree_id, version_id):
156
        object_id = (tree_id, version_id)
157
        self._internal_api.delete(object_id)
158
159
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
160
                         in_signature='a{sv}a{sv}', out_signature='aa{sv}u',
161
                         byte_arrays=True)
162
    def find(self, query_dict, options):
163
        return self._internal_api.find(query_dict, options)
164
165
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
166
                         in_signature='ss', out_signature='s',
167
                         sender_keyword='sender')
168
    def get_data_path(self, tree_id, version_id, sender=None):
169
        object_id = (tree_id, version_id)
170
        return self._internal_api.get_data_path(object_id, sender=sender)
171
172
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
173
                         in_signature='ss', out_signature='a{sv}')
174
    def get_metadata(self, tree_id, version_id):
175
        object_id = (tree_id, version_id)
176
        return self._internal_api.get_properties(object_id)
177
178
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
179
                         in_signature='a{sv}sa{sv}', out_signature='aa{sv}u',
180
                         byte_arrays=True)
181
    def text_search(self, query_dict, query_string, options):
182
        return self._internal_api.find(query_dict, options, query_string)
183
184
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
185
                         in_signature='sssa{sv}s', out_signature='ss',
186
                         async_callbacks=('async_cb', 'async_err_cb'),
187
                         byte_arrays=True)
188
    def restore(self, tree_id, parent_id, version_id, metadata, data_path,
189
                async_cb, async_err_cb):
190
        """
191
        - add a new version with the given ids
192
        - there must be no existing entry with the same (tree_id, version_id)
193
        - if parent_id = '', there must be no existing entry with the same tree_id and no parent_id
194
        """
195
        if not tree_id:
196
            raise ValueError('No tree_id given')
197
198
        metadata['version_id'] = version_id
199
        self._internal_api.save(tree_id, parent_id, metadata, data_path,
200
                                delete_after=True,
201
                                async_cb=async_cb,
202
                                async_err_cb=async_err_cb)
203
204
    def __change_metadata_cb(self, (tree_id, version_id), metadata):
205
        self.ChangedMetadata(tree_id, version_id, metadata)
206
207
    def __delete_cb(self, (tree_id, version_id)):
208
        self.Deleted(tree_id, version_id)
209
210
    def __save_cb(self, tree_id, child_id, parent_id, metadata):
211
        if parent_id:
212
            self.AddedNewVersion(tree_id, child_id, parent_id, metadata)
213
        else:
214
            self.Created(tree_id, child_id, metadata)
215
216
217
class DBusApiSugarV2(dbus.service.Object):
218
    """Compatibility layer for the Sugar 0.84+ data store D-Bus API
219
    """
220
221
    def __init__(self, internal_api):
222
        self._internal_api = internal_api
223
        bus_name = dbus.service.BusName(DBUS_SERVICE_SUGAR_V2,
224
                                        bus=dbus.SessionBus(),
225
                                        replace_existing=False,
226
                                        allow_replacement=False,
227
                                        do_not_queue=True)
228
        dbus.service.Object.__init__(self, bus_name, DBUS_PATH_SUGAR_V2)
229
        self._internal_api.add_callback('change_metadata',
230
                                        self.__change_metadata_cb)
231
        self._internal_api.add_callback('delete', self.__delete_cb)
232
        self._internal_api.add_callback('save', self.__save_cb)
233
234
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
235
                         in_signature='a{sv}sb', out_signature='s',
236
                         async_callbacks=('async_cb', 'async_err_cb'),
237
                         byte_arrays=True)
238
    def create(self, props, file_path, transfer_ownership,
239
               async_cb, async_err_cb):
240
        def success_cb(tree_id, child_id):
241
            async_cb(tree_id)
242
243
        self._internal_api.save(tree_id='', parent_id='', metadata=props,
244
                                path=file_path,
245
                                delete_after=transfer_ownership,
246
                                async_cb=success_cb,
247
                                async_err_cb=async_err_cb)
248
249
    @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='s')
250
    def Created(self, uid):
251
        # pylint: disable-msg=C0103
252
        pass
253
254
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
255
                         in_signature='sa{sv}sb', out_signature='',
256
                         async_callbacks=('async_cb', 'async_err_cb'),
257
                         byte_arrays=True)
258
    def update(self, uid, props, file_path, transfer_ownership,
259
               async_cb, async_err_cb):
260
        def success_cb(tree_id, child_id):
261
            async_cb()
262
263
        latest_versions = self._get_latest(uid)
264
        if not latest_versions:
265
            raise ValueError('Trying to update non-existant entry %s - wanted'
266
                ' to use create()?' % (uid, ))
267
268
        parent = latest_versions[0]
269
        object_id = parent['tree_id'], parent['version_id']
270
        if self._check_identical(parent, file_path):
271
            self._internal_api.change_metadata(object_id, props)
272
            return success_cb(uid, None)
273
274
        self._internal_api.save(tree_id=uid,
275
            parent_id=parent['version_id'], metadata=props,
276
            path=file_path, delete_after=transfer_ownership,
277
            async_cb=success_cb, async_err_cb=async_err_cb)
278
279
    @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='s')
280
    def Updated(self, uid):
281
        # pylint: disable-msg=C0103
282
        pass
283
284
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
285
                         in_signature='a{sv}as', out_signature='aa{sv}u')
286
    def find(self, query, properties):
287
        if 'uid' in properties:
288
            properties.append('tree_id')
289
            properties.remove('uid')
290
291
        options = {'metadata': properties}
292
        for name in ['offset', 'limit', 'order_by']:
293
            if name in query:
294
                options[name] = query.pop(name)
295
296
        if 'uid' in query:
297
            query['tree_id'] = query.pop('uid')
298
299
        results, count = self._internal_api.find(query, options,
300
            query.pop('query', None))
301
302
        if not properties or 'tree_id' in properties:
303
            for entry in results:
304
                entry['uid'] = entry.pop('tree_id')
305
306
        return results, count
307
308
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
309
                         in_signature='s', out_signature='s',
310
                         sender_keyword='sender')
311
    def get_filename(self, uid, sender=None):
312
        latest_versions = self._get_latest(uid)
313
        if not latest_versions:
314
            raise ValueError('Entry %s does not exist' % (uid, ))
315
316
        object_id = (uid, latest_versions[0]['version_id'])
317
        return self._internal_api.get_data_path(object_id, sender=sender)
318
319
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
320
                         in_signature='s', out_signature='a{sv}')
321
    def get_properties(self, uid):
322
        latest_versions = self._get_latest(uid)
323
        if not latest_versions:
324
            raise ValueError('Entry %s does not exist' % (uid, ))
325
326
        latest_versions[0]['uid'] = latest_versions[0].pop('tree_id')
327
        del latest_versions[0]['version_id']
328
        return latest_versions[0]
329
330
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
331
                         in_signature='sa{sv}', out_signature='as')
332
    def get_uniquevaluesfor(self, propertyname, query=None):
333
        return self._internal_api.find_unique_values(query, propertyname)
334
335
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
336
                         in_signature='s', out_signature='')
337
    def delete(self, uid):
338
        latest_versions = self._get_latest(uid)
339
        if not latest_versions:
340
            raise ValueError('Entry %s does not exist' % (uid, ))
341
342
        self._internal_api.delete((uid, latest_versions[0]['version_id']))
343
344
    @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='s')
345
    def Deleted(self, uid):
346
        # pylint: disable-msg=C0103
347
        pass
348
349
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
350
                         in_signature='', out_signature='aa{sv}')
351
    def mounts(self):
352
        return [{'id': 1}]
353
354
    @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='a{sv}')
355
    def Mounted(self, descriptior):
356
        # pylint: disable-msg=C0103
357
        pass
358
359
    @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='a{sv}')
360
    def Unmounted(self, descriptor):
361
        # pylint: disable-msg=C0103
362
        pass
363
364
    def _get_latest(self, uid):
365
        return self._internal_api.find({'tree_id': uid},
366
            {'limit': 1, 'order_by': ['+timestamp']})[0]
367
368
    def _check_identical(self, parent, child_data_path):
369
        """Check whether the new version contains the same data as the parent
370
371
        If child_data_path is empty, but the parent contains data, that's
372
        interpreted as wanting to do a metadata-only update (emulating
373
        sugar-datastore behaviour).
374
        """
375
        parent_object_id = (parent['tree_id'], parent['version_id'])
376
        parent_data_path = self._internal_api.get_data_path(parent_object_id)
377
        if not child_data_path:
378
            return True
379
        elif child_data_path and not parent_data_path:
380
            return False
381
382
        # TODO: compare checksums?
383
        return False
384
385
    def __change_metadata_cb(self, (tree_id, version_id), metadata):
386
        self.Updated(tree_id)
387
388
    def __delete_cb(self, (tree_id, version_id)):
389
        if self._get_latest(tree_id):
390
            self.Updated(tree_id)
391
        else:
392
            self.Deleted(tree_id)
393
394
    def __save_cb(self, tree_id, child_id, parent_id, metadata):
395
        if parent_id:
396
            self.Updated(tree_id)
397
        else:
398
            self.Created(tree_id)
399
400
401
class InternalApi(object):
402
403
    SIGNALS = ['change_metadata', 'delete', 'save']
404
405
    def __init__(self, base_dir):
406
        self._base_dir = base_dir
407
        self._callbacks = {}
408
        self._checkouts_dir = os.path.join(base_dir, 'checkouts')
409
        if not os.path.exists(self._checkouts_dir):
410
            os.makedirs(self._checkouts_dir)
411
        self._git_dir = os.path.join(base_dir, 'git')
412
        self._git_env = {}
413
        gconf_client = gconf.client_get_default()
414
        self._max_versions = gconf_client.get_int(
415
            '/desktop/sugar/datastore/max_versions')
416
        logging.debug('max_versions=%r', self._max_versions)
417
        self._index = Index(os.path.join(self._base_dir, 'index'))
418
        self._migrate()
419
420
    def add_callback(self, signal, callback):
421
        if signal not in InternalApi.SIGNALS:
422
            raise ValueError('Invalid signal %r' % (signal, ))
423
424
        self._callbacks.setdefault(signal, []).append(callback)
425
426
    def change_metadata(self, object_id, metadata):
427
        logging.debug('change_metadata(%r, %r)', object_id, metadata)
428
        metadata['tree_id'], metadata['version_id'] = object_id
429
        if 'creation_time' not in metadata:
430
            old_metadata = self._index.retrieve(object_id)
431
            metadata['creation_time'] = old_metadata['creation_time']
432
433
        self._index.store(object_id, metadata)
434
        self._invoke_callbacks('change_metadata', object_id, metadata)
435
436
    def delete(self, object_id):
437
        logging.debug('delete(%r)', object_id)
438
        self._index.delete(object_id)
439
        self._git_call('update-ref', ['-d', _format_ref(*object_id)])
440
        self._invoke_callbacks('delete', object_id)
441
442
    def get_data_path(self, (tree_id, version_id), sender=None):
443
        logging.debug('get_data_path((%r, %r), %r)', tree_id, version_id,
444
                      sender)
445
        ref_name = _format_ref(tree_id, version_id)
446
        top_level_entries = self._git_call('ls-tree',
447
                                           [ref_name]).splitlines()
448
        if len(top_level_entries) == 1 and \
449
           top_level_entries[0].endswith('\tdata'):
450
            blob_hash = top_level_entries[0].split('\t')[0].split(' ')[2]
451
            return self._checkout_file(blob_hash)
452
453
        return self._checkout_dir(ref_name)
454
455
    def find(self, query_dict, options, query_string=None):
456
        logging.debug('find(%r, %r, %r)', query_dict, options, query_string)
457
        entries, total_count = self._index.find(query_dict, query_string,
458
                                                options)
459
        #logging.debug('object_ids=%r', object_ids)
460
        property_names = options.pop('metadata', None)
461
        for entry in entries:
462
            for name in entry.keys():
463
                if property_names and name not in property_names:
464
                    del entry[name]
465
                elif isinstance(entry[name], str):
466
                    entry[name] = dbus.ByteArray(entry[name])
467
468
        return entries, total_count
469
470
    def find_unique_values(self, query, name):
471
        logging.debug('find_unique_values(%r, %r)', query, name)
472
        if query:
473
            raise NotImplementedError('non-empty query not supported yet')
474
475
        return self._index.find_unique_values(name)
476
477
    def get_properties(self, object_id):
478
        return self._index.retrieve(object_id)
479
480
    def save(self, tree_id, parent_id, metadata, path, delete_after, async_cb,
481
             async_err_cb):
482
        logging.debug('save(%r, %r, %r, %r, %r)', tree_id, parent_id,
483
                      metadata, path, delete_after)
484
485
        if path:
486
            path = os.path.realpath(path)
487
            if not os.access(path, os.R_OK):
488
                raise ValueError('Invalid path given.')
489
490
            if delete_after and not os.access(os.path.dirname(path), os.W_OK):
491
                raise ValueError('Deletion requested for read-only directory')
492
493
        if (not tree_id) and parent_id:
494
            raise ValueError('tree_id is empty but parent_id is not')
495
496
        if tree_id and not parent_id:
497
            if self.find({'tree_id': tree_id}, {'limit': 1})[1]:
498
                raise ValueError('No parent_id given but tree_id already '
499
                                 'exists')
500
501
        if not tree_id:
502
            tree_id = self._gen_uuid()
503
504
        child_id = metadata.get('version_id')
505
        if not child_id:
506
            child_id = self._gen_uuid()
507
        elif not tree_id:
508
            raise ValueError('No tree_id given but metadata contains'
509
                             ' version_id')
510
        elif self._index.contains((tree_id, child_id)):
511
            raise ValueError('There is an existing entry with the same tree_id'
512
                             ' and version_id')
513
514
        if 'timestamp' not in metadata:
515
            metadata['timestamp'] = time.time()
516
517
        if 'creation_time' not in metadata:
518
            metadata['creation_time'] = metadata['timestamp']
519
520
        if os.path.isfile(path):
521
            metadata['filesize'] = str(os.stat(path).st_size)
522
        elif not path:
523
            metadata['filesize'] = '0'
524
525
        tree_id = str(tree_id)
526
        parent_id = str(parent_id)
527
        child_id = str(child_id)
528
529
        metadata['tree_id'] = tree_id
530
        metadata['version_id'] = child_id
531
532
        # TODO: check metadata for validity first (index?)
533
        self._store_entry(tree_id, child_id, parent_id, path, metadata)
534
        self._index.store((tree_id, child_id), metadata)
535
        self._invoke_callbacks('save', tree_id, child_id, parent_id, metadata)
536
537
        if delete_after and path:
538
            os.remove(path)
539
540
        async_cb(tree_id, child_id)
541
542
    def stop(self):
543
        logging.debug('stop()')
544
        self._index.close()
545
546
    def _add_to_index(self, index_path, path):
547
        if os.path.isdir(path):
548
            self._git_call('add', ['-A'], work_dir=path, index_path=index_path)
549
        elif os.path.isfile(path):
550
            object_hash = self._git_call('hash-object', ['-w', path]).strip()
551
            mode = os.stat(path).st_mode
552
            self._git_call('update-index',
553
                           ['--add',
554
                            '--cacheinfo', oct(mode), object_hash, 'data'],
555
                           index_path=index_path)
556
        else:
557
            raise DataStoreError('Refusing to store special object %r' % (path, ))
558
559
    def _check_max_versions(self, tree_id):
560
        if not self._max_versions:
561
            return
562
563
        options = {'all_versions': True, 'offset': self._max_versions,
564
                   'metadata': ['tree_id', 'version_id', 'timestamp'],
565
                   'order_by': ['+timestamp']}
566
        old_versions = self.find({'tree_id': tree_id}, options)[0]
567
        logging.info('Deleting old versions: %r', old_versions)
568
        for entry in old_versions:
569
            self.delete((entry['tree_id'], entry['version_id']))
570
571
    def _checkout_file(self, blob_hash):
572
        fd, file_name = tempfile.mkstemp(dir=self._checkouts_dir)
573
        try:
574
            self._git_call('cat-file', ['blob', blob_hash], stdout_fd=fd)
575
        finally:
576
            os.close(fd)
577
        return file_name
578
579
    def _checkout_dir(self, ref_name):
580
        # FIXME
581
        return ''
582
583
    def _create_repo(self):
584
        os.makedirs(self._git_dir)
585
        self._git_call('init', ['-q', '--bare'])
586
587
    def _find_git_parent(self, tree_id, parent_id):
588
        if not parent_id:
589
            return None
590
591
        try:
592
            return self._git_call('rev-parse',
593
                                  [_format_ref(tree_id, parent_id)]).strip()
594
        except GitError:
595
            return None
596
597
    def _format_commit_message(self, metadata):
598
        return pprint.pformat(to_native(metadata))
599
600
    def _gen_uuid(self):
601
        return str(uuid.uuid4())
602
603
    def _git_call(self, command, args=None, input=None, input_fd=None,
604
                  stdout_fd=None, work_dir=None, index_path=None):
605
        env = dict(self._git_env)
606
        if work_dir:
607
            env['GIT_WORK_TREE'] = work_dir
608
        if index_path:
609
            env['GIT_INDEX_FILE'] = index_path
610
        logging.debug('calling git %s, env=%r', ['git', command] + (args or []), env)
611
        pipe = Popen(['git', command] + (args or []), stdin=input_fd or PIPE,
612
                     stdout=stdout_fd or PIPE, stderr=PIPE, close_fds=True,
613
                     cwd=self._git_dir, env=env)
614
        stdout, stderr = pipe.communicate(input)
615
        if pipe.returncode:
616
            raise GitError(pipe.returncode, stderr)
617
        return stdout
618
619
    def _invoke_callbacks(self, signal, *args):
620
        for callback in self._callbacks.get(signal, []):
621
            callback(*args)
622
623
    def _migrate(self):
624
        if not os.path.exists(self._git_dir):
625
            return self._create_repo()
626
627
    def _store_entry(self, tree_id, version_id, parent_id, path, metadata):
628
        parent_hash = self._find_git_parent(tree_id, parent_id)
629
        commit_message = self._format_commit_message(metadata)
630
        tree_hash = self._write_tree(path)
631
        commit_options = [tree_hash]
632
        if parent_hash:
633
            commit_options += ['-p', parent_hash]
634
        commit_hash = self._git_call('commit-tree', commit_options,
635
                                     input=commit_message).strip()
636
        self._git_call('update-ref', [_format_ref(tree_id, version_id),
637
                                      commit_hash])
638
639
    def _write_tree(self, path):
640
        if not path:
641
            return self._git_call('hash-object',
642
                                  ['-w', '-t', 'tree', '--stdin'],
643
                                  input='').strip()
644
645
        index_dir = tempfile.mkdtemp(prefix='gdatastore-')
646
        index_path = os.path.join(index_dir, 'index')
647
        try:
648
            self._add_to_index(index_path, path)
649
            return self._git_call('write-tree', index_path=index_path).strip()
650
        finally:
651
            shutil.rmtree(index_dir)
652
653
654
def calculate_checksum(path):
655
    checksum = hashlib.sha1()
656
    f = file(path)
657
    while True:
658
        chunk = f.read(65536)
659
        if not chunk:
660
            return checksum.hexdigest()
661
662
        checksum.update(chunk)
663
664
665
def to_native(value):
666
    if isinstance(value, list):
667
        return [to_native(e) for e in value]
668
    elif isinstance(value, dict):
669
        return dict([(to_native(k), to_native(v)) for k, v in value.items()])
670
    elif isinstance(value, unicode):
671
        return unicode(value)
672
    elif isinstance(value, str):
673
        return str(value)
674
    return value
675
676
677
def _format_ref(tree_id, version_id):
678
    return 'refs/gdatastore/%s/%s' % (tree_id, version_id)