Web · Wiki · Activities · Blog · Lists · Chat · Meeting · Bugs · Git · Translate · Archive · People · Donate
c7d8a1b by Sascha Silbe at 2011-04-08 1
#
2
# Author: Sascha Silbe <sascha-pgp@silbe.org>
3
#
4
# This program is free software; you can redistribute it and/or modify
5
# it under the terms of the GNU General Public License version 3
6
# as published by the Free Software Foundation.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program. If not, see <http://www.gnu.org/licenses/>.
15
"""
16
Gdatastore D-Bus service API
17
"""
18
19
import hashlib
20
import logging
f4d2178 by Sascha Silbe at 2011-04-09 21
import os
d8e3d40 by Sascha Silbe at 2011-09-29 22
import pprint
aaa29df by Sascha Silbe at 2011-04-10 23
import shutil
f4d2178 by Sascha Silbe at 2011-04-09 24
from subprocess import Popen, PIPE
aaa29df by Sascha Silbe at 2011-04-10 25
import tempfile
26
import time
c7d8a1b by Sascha Silbe at 2011-04-08 27
import uuid
28
29
import dbus
30
import dbus.service
31
import gconf
32
aaa29df by Sascha Silbe at 2011-04-10 33
from gdatastore.index import Index
c7d8a1b by Sascha Silbe at 2011-04-08 34
35
aaa29df by Sascha Silbe at 2011-04-10 36
DBUS_SERVICE_NATIVE_V1 = 'org.silbe.GDataStore'
37
DBUS_INTERFACE_NATIVE_V1 = 'org.silbe.GDataStore1'
38
DBUS_PATH_NATIVE_V1 = '/org/silbe/GDataStore1'
c7d8a1b by Sascha Silbe at 2011-04-08 39
aaa29df by Sascha Silbe at 2011-04-10 40
DBUS_SERVICE_SUGAR_V2 = 'org.laptop.sugar.DataStore'
41
DBUS_INTERFACE_SUGAR_V2 = 'org.laptop.sugar.DataStore'
42
DBUS_PATH_SUGAR_V2 = '/org/laptop/sugar/DataStore'
43
44
DBUS_SERVICE_SUGAR_V3 = 'org.laptop.sugar.DataStore'
45
DBUS_INTERFACE_SUGAR_V3 = 'org.laptop.sugar.DataStore2'
46
DBUS_PATH_SUGAR_V3 = '/org/laptop/sugar/DataStore2'
47
48
49
class DataStoreError(Exception):
50
    pass
51
52
53
class GitError(DataStoreError):
54
    def __init__(self, returncode, stderr):
55
        self.returncode = returncode
f4d2178 by Sascha Silbe at 2011-04-09 56
        self.stderr = unicode(stderr)
aaa29df by Sascha Silbe at 2011-04-10 57
        Exception.__init__(self)
f4d2178 by Sascha Silbe at 2011-04-09 58
59
    def __unicode__(self):
aaa29df by Sascha Silbe at 2011-04-10 60
        return u'Git returned with exit code #%d: %s' % (self.returncode,
61
                                                         self.stderr)
f4d2178 by Sascha Silbe at 2011-04-09 62
63
    def __str__(self):
64
        return self.__unicode__()
65
66
028e410 by Sascha Silbe at 2011-08-22 67
class DBusApiNativeV1(dbus.service.Object):
68
    """Native gdatastore D-Bus API
69
    """
70
71
    def __init__(self, internal_api):
72
        self._internal_api = internal_api
73
        bus_name = dbus.service.BusName(DBUS_SERVICE_NATIVE_V1,
74
                                        bus=dbus.SessionBus(),
75
                                        replace_existing=False,
76
                                        allow_replacement=False,
77
                                        do_not_queue=True)
78
        dbus.service.Object.__init__(self, bus_name, DBUS_PATH_NATIVE_V1)
79
        self._internal_api.add_callback('change_metadata',
80
                                        self.__change_metadata_cb)
81
        self._internal_api.add_callback('delete', self.__delete_cb)
82
        self._internal_api.add_callback('save', self.__save_cb)
83
84
    @dbus.service.signal(DBUS_INTERFACE_NATIVE_V1, signature='sssa{sv}')
85
    def AddedNewVersion(self, tree_id, child_id, parent_id, metadata):
86
        # pylint: disable-msg=C0103
87
        pass
88
89
    @dbus.service.signal(DBUS_INTERFACE_NATIVE_V1, signature='ssa{sv}')
90
    def Created(self, tree_id, child_id, metadata):
91
        # pylint: disable-msg=C0103
92
        pass
93
94
    @dbus.service.signal(DBUS_INTERFACE_NATIVE_V1, signature='ssa{sv}')
95
    def ChangedMetadata(self, tree_id, version_id, metadata):
96
        # pylint: disable-msg=C0103
97
        pass
98
99
    @dbus.service.signal(DBUS_INTERFACE_NATIVE_V1, signature='ss')
100
    def Deleted(self, tree_id, version_id):
101
        # pylint: disable-msg=C0103
102
        pass
103
104
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
105
                         in_signature='a{sv}s', out_signature='ss',
106
                         async_callbacks=('async_cb', 'async_err_cb'),
107
                         byte_arrays=True)
108
    def create(self, metadata, data_path, async_cb, async_err_cb):
109
        """
110
        - add new entry, assign ids
111
        - data='' indicates no data to store
112
        - bad design? (data OOB)
113
        """
114
        # TODO: what about transfer_ownership/delete_after?
115
        self._internal_api.save(tree_id='', parent_id='', metadata=metadata,
116
                                path=data_path, delete_after=True,
117
                                async_cb=async_cb,
118
                                async_err_cb=async_err_cb)
119
120
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
121
                         in_signature='ssa{sv}s', out_signature='s',
122
                         async_callbacks=('async_cb', 'async_err_cb'),
123
                         byte_arrays=True)
124
    def add_version(self, tree_id, parent_id, metadata, data_path, async_cb,
125
                    async_err_cb):
126
        """
127
        - add new version to existing object
128
        """
129
        def success_cb(tree_id, child_id):
130
            async_cb(child_id)
131
132
        if not tree_id:
133
            raise ValueError('No tree_id given')
134
135
        if not parent_id:
136
            raise ValueError('No parent_id given')
137
138
        self._internal_api.save(tree_id, parent_id, metadata, data_path,
139
                                delete_after=True,
140
                                async_cb=success_cb,
141
                                async_err_cb=async_err_cb)
142
143
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
144
                         in_signature='ssa{sv}', out_signature='',
145
                         byte_arrays=True)
146
    def change_metadata(self, tree_id, version_id, metadata):
147
        """
148
        - change the metadata of an existing version
149
        """
150
        object_id = (tree_id, version_id)
151
        self._internal_api.change_metadata(object_id, metadata)
152
153
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
154
                         in_signature='ss', out_signature='')
155
    def delete(self, tree_id, version_id):
156
        object_id = (tree_id, version_id)
157
        self._internal_api.delete(object_id)
158
159
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
160
                         in_signature='a{sv}a{sv}', out_signature='aa{sv}u',
161
                         byte_arrays=True)
162
    def find(self, query_dict, options):
163
        return self._internal_api.find(query_dict, options)
164
165
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
166
                         in_signature='ss', out_signature='s',
167
                         sender_keyword='sender')
168
    def get_data_path(self, tree_id, version_id, sender=None):
169
        object_id = (tree_id, version_id)
170
        return self._internal_api.get_data_path(object_id, sender=sender)
171
172
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
173
                         in_signature='ss', out_signature='a{sv}')
174
    def get_metadata(self, tree_id, version_id):
175
        object_id = (tree_id, version_id)
176
        return self._internal_api.get_properties(object_id)
177
178
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
179
                         in_signature='a{sv}sa{sv}', out_signature='aa{sv}u',
180
                         byte_arrays=True)
181
    def text_search(self, query_dict, query_string, options):
182
        return self._internal_api.find(query_dict, options, query_string)
183
184
    @dbus.service.method(DBUS_INTERFACE_NATIVE_V1,
7023b15 by Sascha Silbe at 2012-02-20 185
                         in_signature='sssa{sv}s', out_signature='ss',
028e410 by Sascha Silbe at 2011-08-22 186
                         async_callbacks=('async_cb', 'async_err_cb'),
187
                         byte_arrays=True)
188
    def restore(self, tree_id, parent_id, version_id, metadata, data_path,
189
                async_cb, async_err_cb):
190
        """
191
        - add a new version with the given ids
192
        - there must be no existing entry with the same (tree_id, version_id)
193
        - if parent_id = '', there must be no existing entry with the same tree_id and no parent_id
194
        """
195
        if not tree_id:
196
            raise ValueError('No tree_id given')
197
198
        metadata['version_id'] = version_id
199
        self._internal_api.save(tree_id, parent_id, metadata, data_path,
200
                                delete_after=True,
201
                                async_cb=async_cb,
202
                                async_err_cb=async_err_cb)
203
204
    def __change_metadata_cb(self, (tree_id, version_id), metadata):
205
        self.ChangedMetadata(tree_id, version_id, metadata)
206
207
    def __delete_cb(self, (tree_id, version_id)):
208
        self.Deleted(tree_id, version_id)
209
210
    def __save_cb(self, tree_id, child_id, parent_id, metadata):
211
        if parent_id:
212
            self.AddedNewVersion(tree_id, child_id, parent_id, metadata)
213
        else:
214
            self.Created(tree_id, child_id, metadata)
215
216
aaa29df by Sascha Silbe at 2011-04-10 217
class DBusApiSugarV2(dbus.service.Object):
218
    """Compatibility layer for the Sugar 0.84+ data store D-Bus API
c7d8a1b by Sascha Silbe at 2011-04-08 219
    """
220
221
    def __init__(self, internal_api):
222
        self._internal_api = internal_api
aaa29df by Sascha Silbe at 2011-04-10 223
        bus_name = dbus.service.BusName(DBUS_SERVICE_SUGAR_V2,
c7d8a1b by Sascha Silbe at 2011-04-08 224
                                        bus=dbus.SessionBus(),
225
                                        replace_existing=False,
8701947 by Sascha Silbe at 2011-05-15 226
                                        allow_replacement=False,
227
                                        do_not_queue=True)
aaa29df by Sascha Silbe at 2011-04-10 228
        dbus.service.Object.__init__(self, bus_name, DBUS_PATH_SUGAR_V2)
ba4764c by Sascha Silbe at 2011-05-15 229
        self._internal_api.add_callback('change_metadata',
230
                                        self.__change_metadata_cb)
231
        self._internal_api.add_callback('delete', self.__delete_cb)
232
        self._internal_api.add_callback('save', self.__save_cb)
c7d8a1b by Sascha Silbe at 2011-04-08 233
aaa29df by Sascha Silbe at 2011-04-10 234
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
c7d8a1b by Sascha Silbe at 2011-04-08 235
                         in_signature='a{sv}sb', out_signature='s',
236
                         async_callbacks=('async_cb', 'async_err_cb'),
237
                         byte_arrays=True)
238
    def create(self, props, file_path, transfer_ownership,
239
               async_cb, async_err_cb):
240
        def success_cb(tree_id, child_id):
241
            async_cb(tree_id)
242
243
        self._internal_api.save(tree_id='', parent_id='', metadata=props,
244
                                path=file_path,
245
                                delete_after=transfer_ownership,
246
                                async_cb=success_cb,
247
                                async_err_cb=async_err_cb)
248
aaa29df by Sascha Silbe at 2011-04-10 249
    @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='s')
c7d8a1b by Sascha Silbe at 2011-04-08 250
    def Created(self, uid):
251
        # pylint: disable-msg=C0103
252
        pass
253
aaa29df by Sascha Silbe at 2011-04-10 254
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
c7d8a1b by Sascha Silbe at 2011-04-08 255
                         in_signature='sa{sv}sb', out_signature='',
256
                         async_callbacks=('async_cb', 'async_err_cb'),
257
                         byte_arrays=True)
258
    def update(self, uid, props, file_path, transfer_ownership,
259
               async_cb, async_err_cb):
260
        def success_cb(tree_id, child_id):
261
            async_cb()
262
263
        latest_versions = self._get_latest(uid)
264
        if not latest_versions:
265
            raise ValueError('Trying to update non-existant entry %s - wanted'
266
                ' to use create()?' % (uid, ))
267
268
        parent = latest_versions[0]
aaa29df by Sascha Silbe at 2011-04-10 269
        object_id = parent['tree_id'], parent['version_id']
d249b8c by Sascha Silbe at 2011-10-11 270
        if self._check_identical(parent, file_path):
aaa29df by Sascha Silbe at 2011-04-10 271
            self._internal_api.change_metadata(object_id, props)
c7d8a1b by Sascha Silbe at 2011-04-08 272
            return success_cb(uid, None)
273
274
        self._internal_api.save(tree_id=uid,
275
            parent_id=parent['version_id'], metadata=props,
276
            path=file_path, delete_after=transfer_ownership,
277
            async_cb=success_cb, async_err_cb=async_err_cb)
278
aaa29df by Sascha Silbe at 2011-04-10 279
    @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='s')
c7d8a1b by Sascha Silbe at 2011-04-08 280
    def Updated(self, uid):
281
        # pylint: disable-msg=C0103
282
        pass
283
aaa29df by Sascha Silbe at 2011-04-10 284
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
c7d8a1b by Sascha Silbe at 2011-04-08 285
                         in_signature='a{sv}as', out_signature='aa{sv}u')
286
    def find(self, query, properties):
287
        if 'uid' in properties:
288
            properties.append('tree_id')
289
            properties.remove('uid')
290
291
        options = {'metadata': properties}
292
        for name in ['offset', 'limit', 'order_by']:
293
            if name in query:
294
                options[name] = query.pop(name)
295
296
        if 'uid' in query:
297
            query['tree_id'] = query.pop('uid')
298
299
        results, count = self._internal_api.find(query, options,
24f58d5 by Sascha Silbe at 2011-04-10 300
            query.pop('query', None))
c7d8a1b by Sascha Silbe at 2011-04-08 301
aaa29df by Sascha Silbe at 2011-04-10 302
        if not properties or 'tree_id' in properties:
c7d8a1b by Sascha Silbe at 2011-04-08 303
            for entry in results:
304
                entry['uid'] = entry.pop('tree_id')
305
306
        return results, count
307
aaa29df by Sascha Silbe at 2011-04-10 308
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
c7d8a1b by Sascha Silbe at 2011-04-08 309
                         in_signature='s', out_signature='s',
310
                         sender_keyword='sender')
311
    def get_filename(self, uid, sender=None):
312
        latest_versions = self._get_latest(uid)
313
        if not latest_versions:
314
            raise ValueError('Entry %s does not exist' % (uid, ))
315
aaa29df by Sascha Silbe at 2011-04-10 316
        object_id = (uid, latest_versions[0]['version_id'])
317
        return self._internal_api.get_data_path(object_id, sender=sender)
c7d8a1b by Sascha Silbe at 2011-04-08 318
aaa29df by Sascha Silbe at 2011-04-10 319
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
c7d8a1b by Sascha Silbe at 2011-04-08 320
                         in_signature='s', out_signature='a{sv}')
321
    def get_properties(self, uid):
322
        latest_versions = self._get_latest(uid)
323
        if not latest_versions:
324
            raise ValueError('Entry %s does not exist' % (uid, ))
325
326
        latest_versions[0]['uid'] = latest_versions[0].pop('tree_id')
327
        del latest_versions[0]['version_id']
328
        return latest_versions[0]
329
aaa29df by Sascha Silbe at 2011-04-10 330
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
c7d8a1b by Sascha Silbe at 2011-04-08 331
                         in_signature='sa{sv}', out_signature='as')
332
    def get_uniquevaluesfor(self, propertyname, query=None):
333
        return self._internal_api.find_unique_values(query, propertyname)
334
aaa29df by Sascha Silbe at 2011-04-10 335
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
c7d8a1b by Sascha Silbe at 2011-04-08 336
                         in_signature='s', out_signature='')
337
    def delete(self, uid):
338
        latest_versions = self._get_latest(uid)
339
        if not latest_versions:
340
            raise ValueError('Entry %s does not exist' % (uid, ))
341
342
        self._internal_api.delete((uid, latest_versions[0]['version_id']))
343
aaa29df by Sascha Silbe at 2011-04-10 344
    @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='s')
c7d8a1b by Sascha Silbe at 2011-04-08 345
    def Deleted(self, uid):
346
        # pylint: disable-msg=C0103
347
        pass
348
aaa29df by Sascha Silbe at 2011-04-10 349
    @dbus.service.method(DBUS_INTERFACE_SUGAR_V2,
c7d8a1b by Sascha Silbe at 2011-04-08 350
                         in_signature='', out_signature='aa{sv}')
351
    def mounts(self):
352
        return [{'id': 1}]
353
aaa29df by Sascha Silbe at 2011-04-10 354
    @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='a{sv}')
c7d8a1b by Sascha Silbe at 2011-04-08 355
    def Mounted(self, descriptior):
356
        # pylint: disable-msg=C0103
357
        pass
358
aaa29df by Sascha Silbe at 2011-04-10 359
    @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='a{sv}')
c7d8a1b by Sascha Silbe at 2011-04-08 360
    def Unmounted(self, descriptor):
361
        # pylint: disable-msg=C0103
362
        pass
363
364
    def _get_latest(self, uid):
365
        return self._internal_api.find({'tree_id': uid},
366
            {'limit': 1, 'order_by': ['+timestamp']})[0]
367
d249b8c by Sascha Silbe at 2011-10-11 368
    def _check_identical(self, parent, child_data_path):
369
        """Check whether the new version contains the same data as the parent
370
371
        If child_data_path is empty, but the parent contains data, that's
372
        interpreted as wanting to do a metadata-only update (emulating
373
        sugar-datastore behaviour).
374
        """
c7d8a1b by Sascha Silbe at 2011-04-08 375
        parent_object_id = (parent['tree_id'], parent['version_id'])
376
        parent_data_path = self._internal_api.get_data_path(parent_object_id)
d249b8c by Sascha Silbe at 2011-10-11 377
        if not child_data_path:
c7d8a1b by Sascha Silbe at 2011-04-08 378
            return True
d249b8c by Sascha Silbe at 2011-10-11 379
        elif child_data_path and not parent_data_path:
380
            return False
c7d8a1b by Sascha Silbe at 2011-04-08 381
d249b8c by Sascha Silbe at 2011-10-11 382
        # TODO: compare checksums?
aaa29df by Sascha Silbe at 2011-04-10 383
        return False
c7d8a1b by Sascha Silbe at 2011-04-08 384
ba4764c by Sascha Silbe at 2011-05-15 385
    def __change_metadata_cb(self, (tree_id, version_id), metadata):
386
        self.Updated(tree_id)
387
388
    def __delete_cb(self, (tree_id, version_id)):
389
        if self._get_latest(tree_id):
390
            self.Updated(tree_id)
391
        else:
392
            self.Deleted(tree_id)
393
394
    def __save_cb(self, tree_id, child_id, parent_id, metadata):
395
        if parent_id:
396
            self.Updated(tree_id)
397
        else:
398
            self.Created(tree_id)
399
c7d8a1b by Sascha Silbe at 2011-04-08 400
401
class InternalApi(object):
ba4764c by Sascha Silbe at 2011-05-15 402
403
    SIGNALS = ['change_metadata', 'delete', 'save']
404
c7d8a1b by Sascha Silbe at 2011-04-08 405
    def __init__(self, base_dir):
406
        self._base_dir = base_dir
ba4764c by Sascha Silbe at 2011-05-15 407
        self._callbacks = {}
aaa29df by Sascha Silbe at 2011-04-10 408
        self._checkouts_dir = os.path.join(base_dir, 'checkouts')
409
        if not os.path.exists(self._checkouts_dir):
410
            os.makedirs(self._checkouts_dir)
f4d2178 by Sascha Silbe at 2011-04-09 411
        self._git_dir = os.path.join(base_dir, 'git')
aaa29df by Sascha Silbe at 2011-04-10 412
        self._git_env = {}
c7d8a1b by Sascha Silbe at 2011-04-08 413
        gconf_client = gconf.client_get_default()
414
        self._max_versions = gconf_client.get_int(
415
            '/desktop/sugar/datastore/max_versions')
416
        logging.debug('max_versions=%r', self._max_versions)
aaa29df by Sascha Silbe at 2011-04-10 417
        self._index = Index(os.path.join(self._base_dir, 'index'))
f4d2178 by Sascha Silbe at 2011-04-09 418
        self._migrate()
c7d8a1b by Sascha Silbe at 2011-04-08 419
ba4764c by Sascha Silbe at 2011-05-15 420
    def add_callback(self, signal, callback):
421
        if signal not in InternalApi.SIGNALS:
422
            raise ValueError('Invalid signal %r' % (signal, ))
423
424
        self._callbacks.setdefault(signal, []).append(callback)
425
aaa29df by Sascha Silbe at 2011-04-10 426
    def change_metadata(self, object_id, metadata):
427
        logging.debug('change_metadata(%r, %r)', object_id, metadata)
428
        metadata['tree_id'], metadata['version_id'] = object_id
429
        if 'creation_time' not in metadata:
de83019 by Sascha Silbe at 2011-05-15 430
            old_metadata = self._index.retrieve(object_id)
aaa29df by Sascha Silbe at 2011-04-10 431
            metadata['creation_time'] = old_metadata['creation_time']
c7d8a1b by Sascha Silbe at 2011-04-08 432
aaa29df by Sascha Silbe at 2011-04-10 433
        self._index.store(object_id, metadata)
ba4764c by Sascha Silbe at 2011-05-15 434
        self._invoke_callbacks('change_metadata', object_id, metadata)
c7d8a1b by Sascha Silbe at 2011-04-08 435
aaa29df by Sascha Silbe at 2011-04-10 436
    def delete(self, object_id):
437
        logging.debug('delete(%r)', object_id)
438
        self._index.delete(object_id)
439
        self._git_call('update-ref', ['-d', _format_ref(*object_id)])
ba4764c by Sascha Silbe at 2011-05-15 440
        self._invoke_callbacks('delete', object_id)
c7d8a1b by Sascha Silbe at 2011-04-08 441
442
    def get_data_path(self, (tree_id, version_id), sender=None):
443
        logging.debug('get_data_path((%r, %r), %r)', tree_id, version_id,
444
                      sender)
aaa29df by Sascha Silbe at 2011-04-10 445
        ref_name = _format_ref(tree_id, version_id)
446
        top_level_entries = self._git_call('ls-tree',
447
                                           [ref_name]).splitlines()
448
        if len(top_level_entries) == 1 and \
449
           top_level_entries[0].endswith('\tdata'):
450
            blob_hash = top_level_entries[0].split('\t')[0].split(' ')[2]
451
            return self._checkout_file(blob_hash)
452
453
        return self._checkout_dir(ref_name)
454
455
    def find(self, query_dict, options, query_string=None):
456
        logging.debug('find(%r, %r, %r)', query_dict, options, query_string)
457
        entries, total_count = self._index.find(query_dict, query_string,
458
                                                options)
459
        #logging.debug('object_ids=%r', object_ids)
460
        property_names = options.pop('metadata', None)
d2d26fd by Sascha Silbe at 2011-04-10 461
        for entry in entries:
462
            for name in entry.keys():
463
                if property_names and name not in property_names:
464
                    del entry[name]
465
                elif isinstance(entry[name], str):
466
                    entry[name] = dbus.ByteArray(entry[name])
aaa29df by Sascha Silbe at 2011-04-10 467
468
        return entries, total_count
c7d8a1b by Sascha Silbe at 2011-04-08 469
470
    def find_unique_values(self, query, name):
471
        logging.debug('find_unique_values(%r, %r)', query, name)
b0e6e21 by Sascha Silbe at 2011-08-22 472
        if query:
473
            raise NotImplementedError('non-empty query not supported yet')
474
475
        return self._index.find_unique_values(name)
c7d8a1b by Sascha Silbe at 2011-04-08 476
028e410 by Sascha Silbe at 2011-08-22 477
    def get_properties(self, object_id):
478
        return self._index.retrieve(object_id)
479
aaa29df by Sascha Silbe at 2011-04-10 480
    def save(self, tree_id, parent_id, metadata, path, delete_after, async_cb,
481
             async_err_cb):
482
        logging.debug('save(%r, %r, %r, %r, %r)', tree_id, parent_id,
483
                      metadata, path, delete_after)
c7d8a1b by Sascha Silbe at 2011-04-08 484
aaa29df by Sascha Silbe at 2011-04-10 485
        if path:
486
            path = os.path.realpath(path)
487
            if not os.access(path, os.R_OK):
488
                raise ValueError('Invalid path given.')
f4d2178 by Sascha Silbe at 2011-04-09 489
aaa29df by Sascha Silbe at 2011-04-10 490
            if delete_after and not os.access(os.path.dirname(path), os.W_OK):
491
                raise ValueError('Deletion requested for read-only directory')
f4d2178 by Sascha Silbe at 2011-04-09 492
aaa29df by Sascha Silbe at 2011-04-10 493
        if (not tree_id) and parent_id:
494
            raise ValueError('tree_id is empty but parent_id is not')
495
496
        if tree_id and not parent_id:
497
            if self.find({'tree_id': tree_id}, {'limit': 1})[1]:
498
                raise ValueError('No parent_id given but tree_id already '
499
                                 'exists')
500
501
        if not tree_id:
502
            tree_id = self._gen_uuid()
503
504
        child_id = metadata.get('version_id')
505
        if not child_id:
506
            child_id = self._gen_uuid()
507
        elif not tree_id:
508
            raise ValueError('No tree_id given but metadata contains'
509
                             ' version_id')
510
        elif self._index.contains((tree_id, child_id)):
511
            raise ValueError('There is an existing entry with the same tree_id'
512
                             ' and version_id')
513
514
        if 'timestamp' not in metadata:
515
            metadata['timestamp'] = time.time()
516
517
        if 'creation_time' not in metadata:
518
            metadata['creation_time'] = metadata['timestamp']
519
520
        if os.path.isfile(path):
521
            metadata['filesize'] = str(os.stat(path).st_size)
522
        elif not path:
523
            metadata['filesize'] = '0'
524
525
        tree_id = str(tree_id)
526
        parent_id = str(parent_id)
527
        child_id = str(child_id)
528
529
        metadata['tree_id'] = tree_id
530
        metadata['version_id'] = child_id
531
532
        # TODO: check metadata for validity first (index?)
533
        self._store_entry(tree_id, child_id, parent_id, path, metadata)
534
        self._index.store((tree_id, child_id), metadata)
ba4764c by Sascha Silbe at 2011-05-15 535
        self._invoke_callbacks('save', tree_id, child_id, parent_id, metadata)
f34073d by Sascha Silbe at 2011-07-09 536
28bace5 by Sascha Silbe at 2011-08-22 537
        if delete_after and path:
f34073d by Sascha Silbe at 2011-07-09 538
            os.remove(path)
539
aaa29df by Sascha Silbe at 2011-04-10 540
        async_cb(tree_id, child_id)
541
542
    def stop(self):
543
        logging.debug('stop()')
544
        self._index.close()
545
546
    def _add_to_index(self, index_path, path):
547
        if os.path.isdir(path):
548
            self._git_call('add', ['-A'], work_dir=path, index_path=index_path)
549
        elif os.path.isfile(path):
550
            object_hash = self._git_call('hash-object', ['-w', path]).strip()
551
            mode = os.stat(path).st_mode
552
            self._git_call('update-index',
553
                           ['--add',
554
                            '--cacheinfo', oct(mode), object_hash, 'data'],
555
                           index_path=index_path)
556
        else:
557
            raise DataStoreError('Refusing to store special object %r' % (path, ))
f4d2178 by Sascha Silbe at 2011-04-09 558
c7d8a1b by Sascha Silbe at 2011-04-08 559
    def _check_max_versions(self, tree_id):
560
        if not self._max_versions:
561
            return
562
563
        options = {'all_versions': True, 'offset': self._max_versions,
564
                   'metadata': ['tree_id', 'version_id', 'timestamp'],
565
                   'order_by': ['+timestamp']}
566
        old_versions = self.find({'tree_id': tree_id}, options)[0]
567
        logging.info('Deleting old versions: %r', old_versions)
568
        for entry in old_versions:
569
            self.delete((entry['tree_id'], entry['version_id']))
570
aaa29df by Sascha Silbe at 2011-04-10 571
    def _checkout_file(self, blob_hash):
572
        fd, file_name = tempfile.mkstemp(dir=self._checkouts_dir)
573
        try:
574
            self._git_call('cat-file', ['blob', blob_hash], stdout_fd=fd)
575
        finally:
576
            os.close(fd)
577
        return file_name
578
579
    def _checkout_dir(self, ref_name):
580
        # FIXME
581
        return ''
582
583
    def _create_repo(self):
584
        os.makedirs(self._git_dir)
585
        self._git_call('init', ['-q', '--bare'])
586
587
    def _find_git_parent(self, tree_id, parent_id):
588
        if not parent_id:
589
            return None
590
0301a9c by Sascha Silbe at 2012-01-21 591
        try:
592
            return self._git_call('rev-parse',
593
                                  [_format_ref(tree_id, parent_id)]).strip()
594
        except GitError:
595
            return None
aaa29df by Sascha Silbe at 2011-04-10 596
597
    def _format_commit_message(self, metadata):
d8e3d40 by Sascha Silbe at 2011-09-29 598
        return pprint.pformat(to_native(metadata))
aaa29df by Sascha Silbe at 2011-04-10 599
c7d8a1b by Sascha Silbe at 2011-04-08 600
    def _gen_uuid(self):
601
        return str(uuid.uuid4())
602
aaa29df by Sascha Silbe at 2011-04-10 603
    def _git_call(self, command, args=None, input=None, input_fd=None,
604
                  stdout_fd=None, work_dir=None, index_path=None):
605
        env = dict(self._git_env)
606
        if work_dir:
607
            env['GIT_WORK_TREE'] = work_dir
608
        if index_path:
609
            env['GIT_INDEX_FILE'] = index_path
610
        logging.debug('calling git %s, env=%r', ['git', command] + (args or []), env)
611
        pipe = Popen(['git', command] + (args or []), stdin=input_fd or PIPE,
612
                     stdout=stdout_fd or PIPE, stderr=PIPE, close_fds=True,
613
                     cwd=self._git_dir, env=env)
614
        stdout, stderr = pipe.communicate(input)
615
        if pipe.returncode:
616
            raise GitError(pipe.returncode, stderr)
617
        return stdout
618
ba4764c by Sascha Silbe at 2011-05-15 619
    def _invoke_callbacks(self, signal, *args):
620
        for callback in self._callbacks.get(signal, []):
621
            callback(*args)
622
aaa29df by Sascha Silbe at 2011-04-10 623
    def _migrate(self):
624
        if not os.path.exists(self._git_dir):
625
            return self._create_repo()
626
627
    def _store_entry(self, tree_id, version_id, parent_id, path, metadata):
628
        parent_hash = self._find_git_parent(tree_id, parent_id)
629
        commit_message = self._format_commit_message(metadata)
630
        tree_hash = self._write_tree(path)
631
        commit_options = [tree_hash]
632
        if parent_hash:
633
            commit_options += ['-p', parent_hash]
634
        commit_hash = self._git_call('commit-tree', commit_options,
635
                                     input=commit_message).strip()
636
        self._git_call('update-ref', [_format_ref(tree_id, version_id),
637
                                      commit_hash])
638
639
    def _write_tree(self, path):
640
        if not path:
641
            return self._git_call('hash-object',
642
                                  ['-w', '-t', 'tree', '--stdin'],
643
                                  input='').strip()
644
645
        index_dir = tempfile.mkdtemp(prefix='gdatastore-')
646
        index_path = os.path.join(index_dir, 'index')
647
        try:
648
            self._add_to_index(index_path, path)
649
            return self._git_call('write-tree', index_path=index_path).strip()
650
        finally:
651
            shutil.rmtree(index_dir)
652
c7d8a1b by Sascha Silbe at 2011-04-08 653
654
def calculate_checksum(path):
655
    checksum = hashlib.sha1()
656
    f = file(path)
657
    while True:
658
        chunk = f.read(65536)
659
        if not chunk:
660
            return checksum.hexdigest()
661
662
        checksum.update(chunk)
aaa29df by Sascha Silbe at 2011-04-10 663
664
d8e3d40 by Sascha Silbe at 2011-09-29 665
def to_native(value):
666
    if isinstance(value, list):
667
        return [to_native(e) for e in value]
668
    elif isinstance(value, dict):
669
        return dict([(to_native(k), to_native(v)) for k, v in value.items()])
670
    elif isinstance(value, unicode):
671
        return unicode(value)
672
    elif isinstance(value, str):
673
        return str(value)
674
    return value
675
676
aaa29df by Sascha Silbe at 2011-04-10 677
def _format_ref(tree_id, version_id):
678
    return 'refs/gdatastore/%s/%s' % (tree_id, version_id)