Web · Wiki · Activities · Blog · Lists · Chat · Meeting · Bugs · Git · Translate · Archive · People · Donate
1
#
2
# Author: Sascha Silbe <sascha-pgp@silbe.org>
3
#
4
# This program is free software; you can redistribute it and/or modify
5
# it under the terms of the GNU General Public License version 3
6
# as published by the Free Software Foundation.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program. If not, see <http://www.gnu.org/licenses/>.
15
"""
16
Gdatastore D-Bus service API
17
"""
18
19
import hashlib
20
import logging
21
import os
22
import pprint
23
import shutil
24
from subprocess import Popen, PIPE
25
import tempfile
26
import time
27
import uuid
28
29
import dbus
30
import dbus.service
31
import gconf
32
33
from gdatastore.index import Index
34
35
36
DBUS_SERVICE_NATIVE_V1 = 'org.silbe.GDataStore'
37
DBUS_INTERFACE_NATIVE_V1 = 'org.silbe.GDataStore1'
38
DBUS_PATH_NATIVE_V1 = '/org/silbe/GDataStore1'
39
40
DBUS_SERVICE_SUGAR_V2 = 'org.laptop.sugar.DataStore'
41
DBUS_INTERFACE_SUGAR_V2 = 'org.laptop.sugar.DataStore'
42
DBUS_PATH_SUGAR_V2 = '/org/laptop/sugar/DataStore'
43
44
DBUS_SERVICE_SUGAR_V3 = 'org.laptop.sugar.DataStore'
45
DBUS_INTERFACE_SUGAR_V3 = 'org.laptop.sugar.DataStore2'
46
DBUS_PATH_SUGAR_V3 = '/org/laptop/sugar/DataStore2'
47
48
49
class DataStoreError(Exception):
50
    pass
51
52
53
class GitError(DataStoreError):
54
    def __init__(self, returncode, stderr):
55
        self.returncode = returncode
56
        self.stderr = unicode(stderr)
57
        Exception.__init__(self)
58
59
    def __unicode__(self):
60
        return u'Git returned with exit code #%d: %s' % (self.returncode,
61
                                                         self.stderr)
62
63
    def __str__(self):
64
        return self.__unicode__()
65
66
67
class DBusApiNativeV1(dbus.service.Object):
68
    """Native gdatastore D-Bus API
69
    """
70
71
    def __init__(self, internal_api):
72
        self._internal_api = internal_api
73
        bus_name = dbus.service.BusName(DBUS_SERVICE_NATIVE_V1,
74
                                        bus=dbus.SessionBus(),
75
                                        replace_existing=False,
76
                                        allow_replacement=False,
77
                                        do_not_queue=True)
78
        dbus.service.Object.__init__(self, bus_name, DBUS_PATH_NATIVE_V1)
79
        self._internal_api.add_callback('change_metadata',
80
                                        self.__change_metadata_cb)
81
        self._internal_api.add_callback('delete', self.__delete_cb)
82
        self._internal_api.add_callback('save', self.__save_cb)
83
84
    @dbus.service.signal(DBUS_INTERFACE_NATIVE_V1, signature='sssa{sv}')
85
    def AddedNewVersion(self, tree_id, child_id, parent_id, metadata):
86
        # pylint: disable-msg=C0103
87
        pass
88
89
    @dbus.service.signal(DBUS_INTERFACE_NATIVE_V1, signature='ssa{sv}')
90
    def Created(self, tree_id, child_id, metadata):
91
        # pylint: disable-msg=C0103
92
        pass
93
94
    @dbus.service.signal(DBUS_INTERFACE_NATIVE_V1, signature='ssa{sv}')
95
    def ChangedMetadata(self, tree_id, version_id, metadata):
96
        # pylint: disable-msg=C0103
97
        pass
98
99
    @dbus.service.signal(DBUS_INTERFACE_NATIVE_V1, signature='ss')
100
    def Deleted(self, tree_id, version_id):
101
        # pylint: disable-msg=C0103
102
        pass
103
104
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
105
                         in_signature='a{sv}s', out_signature='ss',
106
                         async_callbacks=('async_cb', 'async_err_cb'),
107
                         byte_arrays=True)
108
    def create(self, metadata, data_path, async_cb, async_err_cb):
109
        """
110
        - add new entry, assign ids
111
        - data='' indicates no data to store
112
        - bad design? (data OOB)
113
        """
114
        # TODO: what about transfer_ownership/delete_after?
115
        self._internal_api.save(tree_id='', parent_id='', metadata=metadata,
116
                                path=data_path, delete_after=True,
117
                                async_cb=async_cb,
118
                                async_err_cb=async_err_cb)
119
120
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
121
                         in_signature='ssa{sv}s', out_signature='s',
122
                         async_callbacks=('async_cb', 'async_err_cb'),
123
                         byte_arrays=True)
124
    def add_version(self, tree_id, parent_id, metadata, data_path, async_cb,
125
                    async_err_cb):
126
        """
127
        - add new version to existing object
128
        """
129
        def success_cb(tree_id, child_id):
130
            async_cb(child_id)
131
132
        if not tree_id:
133
            raise ValueError('No tree_id given')
134
135
        if not parent_id:
136
            raise ValueError('No parent_id given')
137
138
        self._internal_api.save(tree_id, parent_id, metadata, data_path,
139
                                delete_after=True,
140
                                async_cb=success_cb,
141
                                async_err_cb=async_err_cb)
142
143
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
144
                         in_signature='ssa{sv}', out_signature='',
145
                         byte_arrays=True)
146
    def change_metadata(self, tree_id, version_id, metadata):
147
        """
148
        - change the metadata of an existing version
149
        """
150
        object_id = (tree_id, version_id)
151
        self._internal_api.change_metadata(object_id, metadata)
152
153
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
154
                         in_signature='ss', out_signature='')
155
    def delete(self, tree_id, version_id):
156
        object_id = (tree_id, version_id)
157
        self._internal_api.delete(object_id)
158
159
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
160
                         in_signature='a{sv}a{sv}', out_signature='aa{sv}u',
161
                         byte_arrays=True)
162
    def find(self, query_dict, options):
163
        return self._internal_api.find(query_dict, options)
164
165
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
166
                         in_signature='ss', out_signature='s',
167
                         sender_keyword='sender')
168
    def get_data_path(self, tree_id, version_id, sender=None):
169
        object_id = (tree_id, version_id)
170
        return self._internal_api.get_data_path(object_id, sender=sender)
171
172
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
173
                         in_signature='ss', out_signature='a{sv}')
174
    def get_metadata(self, tree_id, version_id):
175
        object_id = (tree_id, version_id)
176
        return self._internal_api.get_properties(object_id)
177
178
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
179
                         in_signature='a{sv}sa{sv}', out_signature='aa{sv}u',
180
                         byte_arrays=True)
181
    def text_search(self, query_dict, query_string, options):
182
        return self._internal_api.find(query_dict, options, query_string)
183
184
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
185
                         in_signature='sssa{sv}s', out_signature='ss',
186
                         async_callbacks=('async_cb', 'async_err_cb'),
187
                         byte_arrays=True)
188
    def restore(self, tree_id, parent_id, version_id, metadata, data_path,
189
                async_cb, async_err_cb):
190
        """
191
        - add a new version with the given ids
192
        - there must be no existing entry with the same (tree_id, version_id)
193
        """
194
        if not tree_id:
195
            raise ValueError('No tree_id given')
196
197
        metadata['version_id'] = version_id
198
        self._internal_api.save(tree_id, parent_id, metadata, data_path,
199
                                delete_after=True, allow_new_parent=True,
200
                                async_cb=async_cb,
201
                                async_err_cb=async_err_cb)
202
203
    def __change_metadata_cb(self, (tree_id, version_id), metadata):
204
        self.ChangedMetadata(tree_id, version_id, metadata)
205
206
    def __delete_cb(self, (tree_id, version_id)):
207
        self.Deleted(tree_id, version_id)
208
209
    def __save_cb(self, tree_id, child_id, parent_id, metadata):
210
        if parent_id:
211
            self.AddedNewVersion(tree_id, child_id, parent_id, metadata)
212
        else:
213
            self.Created(tree_id, child_id, metadata)
214
215
216
class DBusApiSugarV2(dbus.service.Object):
217
    """Compatibility layer for the Sugar 0.84+ data store D-Bus API
218
    """
219
220
    def __init__(self, internal_api):
221
        self._internal_api = internal_api
222
        bus_name = dbus.service.BusName(DBUS_SERVICE_SUGAR_V2,
223
                                        bus=dbus.SessionBus(),
224
                                        replace_existing=False,
225
                                        allow_replacement=False,
226
                                        do_not_queue=True)
227
        dbus.service.Object.__init__(self, bus_name, DBUS_PATH_SUGAR_V2)
228
        self._internal_api.add_callback('change_metadata',
229
                                        self.__change_metadata_cb)
230
        self._internal_api.add_callback('delete', self.__delete_cb)
231
        self._internal_api.add_callback('save', self.__save_cb)
232
233
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
234
                         in_signature='a{sv}sb', out_signature='s',
235
                         async_callbacks=('async_cb', 'async_err_cb'),
236
                         byte_arrays=True)
237
    def create(self, props, file_path, transfer_ownership,
238
               async_cb, async_err_cb):
239
        def success_cb(tree_id, child_id):
240
            async_cb(tree_id)
241
242
        self._internal_api.save(tree_id='', parent_id='', metadata=props,
243
                                path=file_path,
244
                                delete_after=transfer_ownership,
245
                                async_cb=success_cb,
246
                                async_err_cb=async_err_cb)
247
248
    @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='s')
249
    def Created(self, uid):
250
        # pylint: disable-msg=C0103
251
        pass
252
253
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
254
                         in_signature='sa{sv}sb', out_signature='',
255
                         async_callbacks=('async_cb', 'async_err_cb'),
256
                         byte_arrays=True)
257
    def update(self, uid, props, file_path, transfer_ownership,
258
               async_cb, async_err_cb):
259
        def success_cb(tree_id, child_id):
260
            async_cb()
261
262
        latest_versions = self._get_latest(uid)
263
        if not latest_versions:
264
            raise ValueError('Trying to update non-existant entry %s - wanted'
265
                ' to use create()?' % (uid, ))
266
267
        parent = latest_versions[0]
268
        object_id = parent['tree_id'], parent['version_id']
269
        if self._check_identical(parent, file_path):
270
            self._internal_api.change_metadata(object_id, props)
271
            return success_cb(uid, None)
272
273
        self._internal_api.save(tree_id=uid,
274
            parent_id=parent['version_id'], metadata=props,
275
            path=file_path, delete_after=transfer_ownership,
276
            async_cb=success_cb, async_err_cb=async_err_cb)
277
278
    @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='s')
279
    def Updated(self, uid):
280
        # pylint: disable-msg=C0103
281
        pass
282
283
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
284
                         in_signature='a{sv}as', out_signature='aa{sv}u')
285
    def find(self, query, properties):
286
        if 'uid' in properties:
287
            properties.append('tree_id')
288
            properties.remove('uid')
289
290
        options = {'metadata': properties}
291
        for name in ['offset', 'limit', 'order_by']:
292
            if name in query:
293
                options[name] = query.pop(name)
294
295
        if 'uid' in query:
296
            query['tree_id'] = query.pop('uid')
297
298
        results, count = self._internal_api.find(query, options,
299
            query.pop('query', None))
300
301
        if not properties or 'tree_id' in properties:
302
            for entry in results:
303
                entry['uid'] = entry.pop('tree_id')
304
305
        return results, count
306
307
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
308
                         in_signature='s', out_signature='s',
309
                         sender_keyword='sender')
310
    def get_filename(self, uid, sender=None):
311
        latest_versions = self._get_latest(uid)
312
        if not latest_versions:
313
            raise ValueError('Entry %s does not exist' % (uid, ))
314
315
        object_id = (uid, latest_versions[0]['version_id'])
316
        return self._internal_api.get_data_path(object_id, sender=sender)
317
318
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
319
                         in_signature='s', out_signature='a{sv}')
320
    def get_properties(self, uid):
321
        latest_versions = self._get_latest(uid)
322
        if not latest_versions:
323
            raise ValueError('Entry %s does not exist' % (uid, ))
324
325
        latest_versions[0]['uid'] = latest_versions[0].pop('tree_id')
326
        del latest_versions[0]['version_id']
327
        return latest_versions[0]
328
329
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
330
                         in_signature='sa{sv}', out_signature='as')
331
    def get_uniquevaluesfor(self, propertyname, query=None):
332
        return self._internal_api.find_unique_values(query, propertyname)
333
334
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
335
                         in_signature='s', out_signature='')
336
    def delete(self, uid):
337
        latest_versions = self._get_latest(uid)
338
        if not latest_versions:
339
            raise ValueError('Entry %s does not exist' % (uid, ))
340
341
        self._internal_api.delete((uid, latest_versions[0]['version_id']))
342
343
    @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='s')
344
    def Deleted(self, uid):
345
        # pylint: disable-msg=C0103
346
        pass
347
348
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
349
                         in_signature='', out_signature='aa{sv}')
350
    def mounts(self):
351
        return [{'id': 1}]
352
353
    @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='a{sv}')
354
    def Mounted(self, descriptior):
355
        # pylint: disable-msg=C0103
356
        pass
357
358
    @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='a{sv}')
359
    def Unmounted(self, descriptor):
360
        # pylint: disable-msg=C0103
361
        pass
362
363
    def _get_latest(self, uid):
364
        return self._internal_api.find({'tree_id': uid},
365
            {'limit': 1, 'order_by': ['+timestamp']})[0]
366
367
    def _check_identical(self, parent, child_data_path):
368
        """Check whether the new version contains the same data as the parent
369
370
        If child_data_path is empty, but the parent contains data, that's
371
        interpreted as wanting to do a metadata-only update (emulating
372
        sugar-datastore behaviour).
373
        """
374
        parent_object_id = (parent['tree_id'], parent['version_id'])
375
        parent_data_path = self._internal_api.get_data_path(parent_object_id)
376
        if not child_data_path:
377
            return True
378
        elif child_data_path and not parent_data_path:
379
            return False
380
381
        # TODO: compare checksums?
382
        return False
383
384
    def __change_metadata_cb(self, (tree_id, version_id), metadata):
385
        self.Updated(tree_id)
386
387
    def __delete_cb(self, (tree_id, version_id)):
388
        if self._get_latest(tree_id):
389
            self.Updated(tree_id)
390
        else:
391
            self.Deleted(tree_id)
392
393
    def __save_cb(self, tree_id, child_id, parent_id, metadata):
394
        if parent_id:
395
            self.Updated(tree_id)
396
        else:
397
            self.Created(tree_id)
398
399
400
class InternalApi(object):
401
402
    SIGNALS = ['change_metadata', 'delete', 'save']
403
404
    def __init__(self, base_dir):
405
        self._base_dir = base_dir
406
        self._callbacks = {}
407
        self._checkouts_dir = os.path.join(base_dir, 'checkouts')
408
        if not os.path.exists(self._checkouts_dir):
409
            os.makedirs(self._checkouts_dir)
410
        self._git_dir = os.path.join(base_dir, 'git')
411
        self._git_env = {}
412
        gconf_client = gconf.client_get_default()
413
        self._max_versions = gconf_client.get_int(
414
            '/desktop/sugar/datastore/max_versions')
415
        logging.debug('max_versions=%r', self._max_versions)
416
        self._index = Index(os.path.join(self._base_dir, 'index'))
417
        self._migrate()
418
419
    def add_callback(self, signal, callback):
420
        if signal not in InternalApi.SIGNALS:
421
            raise ValueError('Invalid signal %r' % (signal, ))
422
423
        self._callbacks.setdefault(signal, []).append(callback)
424
425
    def change_metadata(self, object_id, metadata):
426
        logging.debug('change_metadata(%r, %r)', object_id, metadata)
427
        metadata['tree_id'], metadata['version_id'] = object_id
428
        if 'creation_time' not in metadata:
429
            old_metadata = self._index.retrieve(object_id)
430
            metadata['creation_time'] = old_metadata['creation_time']
431
432
        self._index.store(object_id, metadata)
433
        self._invoke_callbacks('change_metadata', object_id, metadata)
434
435
    def delete(self, object_id):
436
        logging.debug('delete(%r)', object_id)
437
        self._index.delete(object_id)
438
        self._git_call('update-ref', ['-d', _format_ref(*object_id)])
439
        self._invoke_callbacks('delete', object_id)
440
441
    def get_data_path(self, (tree_id, version_id), sender=None):
442
        logging.debug('get_data_path((%r, %r), %r)', tree_id, version_id,
443
                      sender)
444
        ref_name = _format_ref(tree_id, version_id)
445
        top_level_entries = self._git_call('ls-tree',
446
                                           [ref_name]).splitlines()
447
        if len(top_level_entries) == 1 and \
448
           top_level_entries[0].endswith('\tdata'):
449
            blob_hash = top_level_entries[0].split('\t')[0].split(' ')[2]
450
            return self._checkout_file(blob_hash)
451
452
        return self._checkout_dir(ref_name)
453
454
    def find(self, query_dict, options, query_string=None):
455
        logging.debug('find(%r, %r, %r)', query_dict, options, query_string)
456
        entries, total_count = self._index.find(query_dict, query_string,
457
                                                options)
458
        #logging.debug('object_ids=%r', object_ids)
459
        property_names = options.pop('metadata', None)
460
        for entry in entries:
461
            for name in entry.keys():
462
                if property_names and name not in property_names:
463
                    del entry[name]
464
                elif isinstance(entry[name], str):
465
                    entry[name] = dbus.ByteArray(entry[name])
466
467
        return entries, total_count
468
469
    def find_unique_values(self, query, name):
470
        logging.debug('find_unique_values(%r, %r)', query, name)
471
        if query:
472
            raise NotImplementedError('non-empty query not supported yet')
473
474
        return self._index.find_unique_values(name)
475
476
    def get_properties(self, object_id):
477
        return self._index.retrieve(object_id)
478
479
    def save(self, tree_id, parent_id, metadata, path, delete_after, async_cb,
480
             async_err_cb, allow_new_parent=False):
481
        logging.debug('save(%r, %r, %r, %r, %r)', tree_id, parent_id,
482
                      metadata, path, delete_after)
483
484
        if path:
485
            path = os.path.realpath(path)
486
            if not os.access(path, os.R_OK):
487
                raise ValueError('Invalid path given.')
488
489
            if delete_after and not os.access(os.path.dirname(path), os.W_OK):
490
                raise ValueError('Deletion requested for read-only directory')
491
492
        if (not tree_id) and parent_id:
493
            raise ValueError('tree_id is empty but parent_id is not')
494
495
        if tree_id and not parent_id and not allow_new_parent:
496
            if self.find({'tree_id': tree_id}, {'limit': 1})[1]:
497
                raise ValueError('No parent_id given but tree_id already '
498
                                 'exists')
499
500
        if not tree_id:
501
            tree_id = self._gen_uuid()
502
503
        child_id = metadata.get('version_id')
504
        if not child_id:
505
            child_id = self._gen_uuid()
506
        elif not tree_id:
507
            raise ValueError('No tree_id given but metadata contains'
508
                             ' version_id')
509
        elif self._index.contains((tree_id, child_id)):
510
            raise ValueError('There is an existing entry with the same tree_id'
511
                             ' and version_id')
512
513
        if 'timestamp' not in metadata:
514
            metadata['timestamp'] = time.time()
515
516
        if 'creation_time' not in metadata:
517
            metadata['creation_time'] = metadata['timestamp']
518
519
        if os.path.isfile(path):
520
            metadata['filesize'] = str(os.stat(path).st_size)
521
        elif not path:
522
            metadata['filesize'] = '0'
523
524
        tree_id = str(tree_id)
525
        parent_id = str(parent_id)
526
        child_id = str(child_id)
527
528
        metadata['tree_id'] = tree_id
529
        metadata['version_id'] = child_id
530
531
        # TODO: check metadata for validity first (index?)
532
        self._store_entry(tree_id, child_id, parent_id, path, metadata)
533
        self._index.store((tree_id, child_id), metadata)
534
        self._invoke_callbacks('save', tree_id, child_id, parent_id, metadata)
535
536
        if delete_after and path:
537
            os.remove(path)
538
539
        async_cb(tree_id, child_id)
540
541
    def stop(self):
542
        logging.debug('stop()')
543
        self._index.close()
544
545
    def _add_to_index(self, index_path, path):
546
        if os.path.isdir(path):
547
            self._git_call('add', ['-A'], work_dir=path, index_path=index_path)
548
        elif os.path.isfile(path):
549
            object_hash = self._git_call('hash-object', ['-w', path]).strip()
550
            mode = os.stat(path).st_mode
551
            self._git_call('update-index',
552
                           ['--add',
553
                            '--cacheinfo', oct(mode), object_hash, 'data'],
554
                           index_path=index_path)
555
        else:
556
            raise DataStoreError('Refusing to store special object %r' % (path, ))
557
558
    def _check_max_versions(self, tree_id):
559
        if not self._max_versions:
560
            return
561
562
        options = {'all_versions': True, 'offset': self._max_versions,
563
                   'metadata': ['tree_id', 'version_id', 'timestamp'],
564
                   'order_by': ['+timestamp']}
565
        old_versions = self.find({'tree_id': tree_id}, options)[0]
566
        logging.info('Deleting old versions: %r', old_versions)
567
        for entry in old_versions:
568
            self.delete((entry['tree_id'], entry['version_id']))
569
570
    def _checkout_file(self, blob_hash):
571
        fd, file_name = tempfile.mkstemp(dir=self._checkouts_dir)
572
        try:
573
            self._git_call('cat-file', ['blob', blob_hash], stdout_fd=fd)
574
        finally:
575
            os.close(fd)
576
        return file_name
577
578
    def _checkout_dir(self, ref_name):
579
        # FIXME
580
        return ''
581
582
    def _create_repo(self):
583
        os.makedirs(self._git_dir)
584
        self._git_call('init', ['-q', '--bare'])
585
586
    def _format_commit_message(self, metadata):
587
        return pprint.pformat(to_native(metadata))
588
589
    def _gen_uuid(self):
590
        return str(uuid.uuid4())
591
592
    def _git_call(self, command, args=None, input=None, input_fd=None,
593
                  stdout_fd=None, work_dir=None, index_path=None):
594
        env = dict(self._git_env)
595
        if work_dir:
596
            env['GIT_WORK_TREE'] = work_dir
597
        if index_path:
598
            env['GIT_INDEX_FILE'] = index_path
599
        logging.debug('calling git %s, env=%r', ['git', command] + (args or []), env)
600
        pipe = Popen(['git', command] + (args or []), stdin=input_fd or PIPE,
601
                     stdout=stdout_fd or PIPE, stderr=PIPE, close_fds=True,
602
                     cwd=self._git_dir, env=env)
603
        stdout, stderr = pipe.communicate(input)
604
        if pipe.returncode:
605
            raise GitError(pipe.returncode, stderr)
606
        return stdout
607
608
    def _invoke_callbacks(self, signal, *args):
609
        for callback in self._callbacks.get(signal, []):
610
            callback(*args)
611
612
    def _migrate(self):
613
        if not os.path.exists(self._git_dir):
614
            return self._create_repo()
615
616
    def _store_entry(self, tree_id, version_id, parent_id, path, metadata):
617
        commit_message = self._format_commit_message(metadata)
618
        tree_hash = self._write_tree(path)
619
        commit_hash = self._git_call('commit-tree', [tree_hash],
620
                                     input=commit_message).strip()
621
        self._git_call('update-ref', [_format_ref(tree_id, version_id),
622
                                      commit_hash])
623
624
    def _write_tree(self, path):
625
        if not path:
626
            return self._git_call('hash-object',
627
                                  ['-w', '-t', 'tree', '--stdin'],
628
                                  input='').strip()
629
630
        index_dir = tempfile.mkdtemp(prefix='gdatastore-')
631
        index_path = os.path.join(index_dir, 'index')
632
        try:
633
            self._add_to_index(index_path, path)
634
            return self._git_call('write-tree', index_path=index_path).strip()
635
        finally:
636
            shutil.rmtree(index_dir)
637
638
639
def calculate_checksum(path):
640
    checksum = hashlib.sha1()
641
    f = file(path)
642
    while True:
643
        chunk = f.read(65536)
644
        if not chunk:
645
            return checksum.hexdigest()
646
647
        checksum.update(chunk)
648
649
650
def to_native(value):
651
    if isinstance(value, list):
652
        return [to_native(e) for e in value]
653
    elif isinstance(value, dict):
654
        return dict([(to_native(k), to_native(v)) for k, v in value.items()])
655
    elif isinstance(value, unicode):
656
        return unicode(value)
657
    elif isinstance(value, str):
658
        return str(value)
659
    elif isinstance(value, int):
660
        return int(value)
661
    elif isinstance(value, float):
662
        return float(value)
663
    else:
664
        raise TypeError('Unknown type: %s' % (type(value), ))
665
666
667
def _format_ref(tree_id, version_id):
668
    return 'refs/gdatastore/%s/%s' % (tree_id, version_id)