Web · Wiki · Activities · Blog · Lists · Chat · Meeting · Bugs · Git · Translate · Archive · People · Donate
1
#
2
# Author: Sascha Silbe <sascha-pgp@silbe.org> (OpenPGP signed mails only)
3
#
4
# This program is free software; you can redistribute it and/or modify
5
# it under the terms of the GNU General Public License version 3
6
# as published by the Free Software Foundation.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program. If not, see <http://www.gnu.org/licenses/>.
15
import collections
16
import errno
17
import functools
18
import logging
19
import os
20
import stat
21
import threading
22
import time
23
24
import dbus
25
26
import sugar.mime
27
28
29
DS_DBUS_SERVICE = 'org.laptop.sugar.DataStore'
30
DS_DBUS_INTERFACE1 = 'org.laptop.sugar.DataStore'
31
DS_DBUS_PATH1 = '/org/laptop/sugar/DataStore'
32
DS_DBUS_INTERFACE2 = 'org.laptop.sugar.DataStore2'
33
DS_DBUS_PATH2 = '/org/laptop/sugar/DataStore2'
34
35
# nearly infinite
36
DBUS_TIMEOUT_MAX = 2 ** 31 / 1000
37
DBUS_PYTHON_VALUE_ERROR = 'org.freedesktop.DBus.Python.ValueError'
38
39
_USEFUL_PROPS = ['mime_type', 'tags', 'timestamp', 'title']
40
"""Metadata properties used for determining the file name of an entry"""
41
42
43
def synchronised(func):
44
    @functools.wraps(func)
45
    def wrapper(self, *args, **kwargs):
46
        with self._lock:
47
            return func(self, *args, **kwargs)
48
    return wrapper
49
50
51
class _LRU(collections.MutableMapping):
52
    """Simple, but reasonably fast Least Recently Used (LRU) cache"""
53
54
    def __init__(self, capacity):
55
        self.capacity = capacity
56
        self._dict = {}
57
        self._q = collections.deque()
58
        self.__contains__ = self._dict.__contains__
59
60
    def __delitem__(self, key):
61
        self._q.remove(key)
62
        del self._dict[key]
63
64
    def __iter__(self):
65
        return self._dict.__iter__()
66
67
    def __getitem__(self, key):
68
        value = self._dict[key]
69
        if self._q[-1] == key:
70
            return value
71
72
        self._q.remove(key)
73
        self._q.append(key)
74
        return value
75
76
    def __len__(self):
77
        return len(self._q)
78
79
    def __setitem__(self, key, value):
80
        if key in self._dict:
81
            self._q.remove(key)
82
        elif len(self._dict) == self.capacity:
83
            del self._dict[self._q.popleft()]
84
85
        self._q.append(key)
86
        self._dict[key] = value
87
88
    def clear(self):
89
        self._q.clear()
90
        self._dict.clear()
91
92
93
class DataStore(object):
94
    def __init__(self):
95
        self.supports_versions = False
96
        self._lock = threading.RLock()
97
        self._data_store_version = 0
98
        bus = dbus.SessionBus()
99
        try:
100
            self._data_store = dbus.Interface(bus.get_object(DS_DBUS_SERVICE,
101
                DS_DBUS_PATH2), DS_DBUS_INTERFACE2)
102
            self._data_store.find({'tree_id': 'invalid'},
103
                {'metadata': ['tree_id']})
104
            self.supports_versions = True
105
            logging.info('Data store with version support found')
106
            return
107
108
        except dbus.DBusException:
109
            logging.debug('No data store with version support found')
110
111
        self._data_store = dbus.Interface(bus.get_object(DS_DBUS_SERVICE,
112
            DS_DBUS_PATH1), DS_DBUS_INTERFACE1)
113
        self._data_store.find({'uid': 'invalid'}, ['uid'])
114
        logging.info('Data store without version support found')
115
116
        if 'uri' in self._data_store.mounts()[0]:
117
            self._data_store_version = 82
118
            data_store_path = '/home/olpc/.sugar/default/datastore'
119
            self._data_store_mount_id = [mount['id']
120
                for mount in self._data_store.mounts()
121
                if mount['uri'] == data_store_path][0]
122
            logging.info('0.82 data store found')
123
        else:
124
            logging.info('0.84+ data store without version support found')
125
            self._data_store_version = 84
126
127
    @synchronised
128
    def list_object_ids(self, query=None):
129
        """Retrieve the object_ids of all (matching) data store entries
130
131
        Only return the latest version of each entry for data stores with
132
        version support.
133
        """
134
        query = query or {}
135
        if self._data_store.dbus_interface == DS_DBUS_INTERFACE2:
136
            options = {'metadata': ['tree_id', 'version_id'],
137
                       'order_by': ['-timestamp']}
138
            return [(unicode(entry['tree_id']), unicode(entry['version_id']))
139
                    for entry in self._data_store.find(query, options,
140
                        timeout=DBUS_TIMEOUT_MAX, byte_arrays=True)[0]]
141
142
        elif self._data_store_version == 82:
143
            properties = ['uid', 'mountpoint']
144
            return [unicode(entry['uid'])
145
                    for entry in self._data_store.find(query, properties,
146
                        byte_arrays=True, timeout=DBUS_TIMEOUT_MAX)[0]
147
                    if entry['mountpoint'] == self._data_store_mount_id]
148
149
        else:
150
            return [unicode(entry['uid'])
151
                    for entry in self._data_store.find(query, ['uid'],
152
                        byte_arrays=True, timeout=DBUS_TIMEOUT_MAX)[0]]
153
154
    @synchronised
155
    def list_metadata(self, query=None):
156
        """Retrieve object_id and selected metadata of matching entries
157
158
        Only return the latest version of each entry for data stores with
159
        version support.
160
161
        Returns a list of tuples containing the object_id and metadata.
162
        """
163
        query = query or {}
164
165
        properties = list(_USEFUL_PROPS)
166
        if self._data_store.dbus_interface == DS_DBUS_INTERFACE2:
167
            properties += ['parent_id', 'tree_id', 'version_id']
168
            options = {'metadata': properties,
169
                       'all_versions': True, 'order_by': ['-timestamp']}
170
            return [((unicode(entry['tree_id']),
171
                      unicode(entry['version_id'])),
172
                     self._convert_metadata(entry))
173
                    for entry in self._data_store.find(query, options,
174
                        timeout=DBUS_TIMEOUT_MAX, byte_arrays=True)[0]]
175
176
        elif self._data_store_version == 82:
177
            properties += ['uid', 'mountpoint']
178
            return [(unicode(entry['uid']), self._convert_metadata(entry))
179
                    for entry in self._data_store.find(query, properties,
180
                        timeout=DBUS_TIMEOUT_MAX, byte_arrays=True)[0]
181
                    if entry['mountpoint'] == self._data_store_mount_id]
182
183
        else:
184
            properties += ['uid']
185
            return [(unicode(entry['uid']), self._convert_metadata(entry))
186
                    for entry in self._data_store.find(query, properties,
187
                        timeout=DBUS_TIMEOUT_MAX, byte_arrays=True)[0]]
188
189
    @synchronised
190
    def list_versions(self, tree_id):
191
        """Retrieve all version_ids of the given data store entry"""
192
        options = {'all_versions': True, 'order_by': ['-timestamp']}
193
        return [unicode(entry['version_id'])
194
                for entry in self._data_store.find({'tree_id': tree_id},
195
                    options, timeout=DBUS_TIMEOUT_MAX, byte_arrays=True)[0]]
196
197
    @synchronised
198
    def list_tree_ids(self, query=None):
199
        """Retrieve the tree_ids of all (matching) data store entries"""
200
        return [unicode(entry[0]) for entry in self.list_object_ids(query)]
201
202
    @synchronised
203
    def list_property_values(self, name, query=None):
204
        """Return all unique values of the given property"""
205
        assert isinstance(name, unicode)
206
207
        query = query or {}
208
        if self._data_store.dbus_interface == DS_DBUS_INTERFACE2:
209
            options = {'metadata': [name], 'all_versions': True}
210
            entries = self._data_store.find(query, options,
211
                                            timeout=DBUS_TIMEOUT_MAX,
212
                                            byte_arrays=True)[0]
213
        else:
214
            # We can't use get_uniquevaluesfor() as sugar-datastore
215
            # only supports it for activity_id, which is not what
216
            # we need.
217
            entries = self._data_store.find(query, [name],
218
                                            timeout=DBUS_TIMEOUT_MAX,
219
                                            byte_arrays=True)[0]
220
221
        return dict.fromkeys([entry.get(name) for entry in entries]).keys()
222
223
    @synchronised
224
    def check_object_id(self, object_id):
225
        """Return True if the given object_id identifies a data store entry"""
226
        try:
227
            self.get_properties(object_id, [u'uid'])
228
        except dbus.DBusException, exception:
229
            if exception.get_dbus_name() == DBUS_PYTHON_VALUE_ERROR:
230
                return False
231
            raise
232
233
        return True
234
235
    @synchronised
236
    def check_tree_id(self, tree_id):
237
        """Return True if the given tree_id identifies a data store entry"""
238
        assert isinstance(tree_id, unicode)
239
        results = self._data_store.find({'tree_id': tree_id}, {},
240
                                        timeout=DBUS_TIMEOUT_MAX,
241
                                        byte_arrays=True)[0]
242
        return bool(results)
243
244
    @synchronised
245
    def check_property_contains(self, name, word):
246
        """Return True if there is at least one entry containing word in the
247
        given property
248
        """
249
        assert isinstance(name, unicode)
250
        assert isinstance(word, unicode)
251
252
        query_string = u'%s:"%s"' % (name, word.replace(u'"', u''))
253
        if self._data_store.dbus_interface == DS_DBUS_INTERFACE2:
254
            options = {'limit': 1}
255
            results = self._data_store.text_search({}, query_string, options,
256
                                                   timeout=DBUS_TIMEOUT_MAX,
257
                                                   byte_arrays=True)[0]
258
        else:
259
            query = {'query': query_string, 'limit': 1}
260
            results = self._data_store.find(query, [name],
261
                                            timeout=DBUS_TIMEOUT_MAX,
262
                                            byte_arrays=True)[0]
263
264
        return bool(results)
265
266
    @synchronised
267
    def get_properties(self, object_id, names=None):
268
        """Read given properties for data store entry identified by object_id
269
270
        Returns a dictionary with unicode strings as keys and values.
271
        """
272
        if self._data_store.dbus_interface == DS_DBUS_INTERFACE2:
273
            tree_id, version_id = object_id
274
            assert isinstance(tree_id, unicode)
275
            assert isinstance(version_id, unicode)
276
277
            query = {'tree_id': tree_id, 'version_id': version_id}
278
            options = {}
279
            if names:
280
                options['metadata'] = names
281
282
            results = self._data_store.find(query, options,
283
                                            timeout=DBUS_TIMEOUT_MAX,
284
                                            byte_arrays=True)[0]
285
            if not results:
286
                raise ValueError('Object %r does not exist' % (object_id, ))
287
288
            return self._convert_metadata(results[0])
289
290
        else:
291
            assert isinstance(object_id, unicode)
292
293
            metadata = self._data_store.get_properties(object_id,
294
                byte_arrays=True)
295
            metadata['uid'] = object_id
296
            if names:
297
                metadata = dict([(name, metadata[name]) for name in names
298
                                 if name in metadata])
299
300
            return self._convert_metadata(metadata)
301
302
    @synchronised
303
    def list_properties(self, object_id):
304
        """List the names of all properties for this entry
305
306
        Returns a list of unicode strings.
307
        """
308
        return self.get_properties(object_id).keys()
309
310
    @synchronised
311
    def create_property(self, object_id, name, value):
312
        """Set the given property, raising an error if it already exists"""
313
        assert isinstance(name, unicode)
314
315
        metadata = self.get_properties(object_id)
316
        if name in metadata:
317
            raise IOError(errno.EEXIST, os.strerror(errno.EEXIST))
318
        metadata[name] = value
319
        self._change_metadata(object_id, metadata)
320
321
    @synchronised
322
    def replace_property(self, object_id, name, value):
323
        """Modify the given, already existing property"""
324
        assert isinstance(name, unicode)
325
        assert isinstance(value, unicode)
326
327
        metadata = self.get_properties(object_id)
328
        if name not in metadata:
329
            # on Linux ENOATTR=ENODATA (Python errno doesn't contain ENOATTR)
330
            raise IOError(errno.ENODATA, os.strerror(errno.ENODATA))
331
        metadata[name] = value
332
        self._change_metadata(object_id, metadata)
333
334
    @synchronised
335
    def set_properties(self, object_id, properties):
336
        """Write the given (sub)set of properties
337
338
        properties -- metadata as dictionary with unicode strings as
339
                      keys and values
340
        """
341
        assert not [True for key, value in properties.items()
342
                    if (not isinstance(key, unicode)) or
343
                       (not isinstance(value, unicode))]
344
345
        metadata = self.get_properties(object_id)
346
        metadata.update(properties)
347
        self._change_metadata(object_id, metadata)
348
349
    @synchronised
350
    def remove_properties(self, object_id, names):
351
        """Remove the given (sub)set of properties
352
353
        names -- list of property names (unicode strings)
354
        """
355
        metadata = self.get_properties(object_id)
356
        for name in names:
357
            assert isinstance(name, unicode)
358
359
            if name not in metadata:
360
                # on Linux ENOATTR=ENODATA (and no ENOATTR in errno module)
361
                raise IOError(errno.ENODATA, os.strerror(errno.ENODATA))
362
            del metadata[name]
363
364
        self._change_metadata(object_id, metadata)
365
366
    @synchronised
367
    def remove_entry(self, object_id):
368
        """Remove a single (version of a) data store entry"""
369
        if self._data_store.dbus_interface == DS_DBUS_INTERFACE2:
370
            tree_id, version_id = object_id
371
            assert isinstance(tree_id, unicode)
372
            assert isinstance(version_id, unicode)
373
374
            self._data_store.delete(tree_id, version_id,
375
                                    timeout=DBUS_TIMEOUT_MAX)
376
377
        else:
378
            assert isinstance(object_id, unicode)
379
            self._data_store.delete(object_id, timeout=DBUS_TIMEOUT_MAX)
380
381
    @synchronised
382
    def create_new(self, properties):
383
        """Create a new data store entry
384
385
        properties -- metadata as dictionary with unicode strings as
386
                      keys and values
387
        """
388
        assert not [True for key, value in properties.items()
389
                    if (not isinstance(key, unicode)) or
390
                       (not isinstance(value, unicode))]
391
392
        if self._data_store.dbus_interface == DS_DBUS_INTERFACE2:
393
            return self._data_store.save('', '', properties, '', False,
394
                                         timeout=DBUS_TIMEOUT_MAX)
395
396
        else:
397
            return self._data_store.create(properties, '', False)
398
399
    @synchronised
400
    def get_data(self, object_id):
401
        """Return path to data for data store entry identified by object_id."""
402
        if self._data_store.dbus_interface == DS_DBUS_INTERFACE2:
403
            tree_id, version_id = object_id
404
            assert isinstance(tree_id, unicode)
405
            assert isinstance(version_id, unicode)
406
            return self._data_store.get_data(tree_id, version_id,
407
                byte_arrays=True)
408
409
        else:
410
            assert isinstance(object_id, unicode)
411
            return self._data_store.get_filename(object_id, byte_arrays=True)
412
413
    @synchronised
414
    def get_size(self, object_id):
415
        # FIXME: make use of filesize property if available
416
        path = self.get_data(object_id)
417
        if not path:
418
            return 0
419
420
        size = os.stat(path).st_size
421
        os.remove(path)
422
        return size
423
424
    @synchronised
425
    def write_data(self, object_id, path):
426
        """Update data for data store entry identified by object_id.
427
428
        Return object_id of the updated entry. If the data store does not
429
        support versions, this will be the same as the one given as parameter.
430
431
        path -- Path of data file in real file system (string)
432
        """
433
        assert isinstance(path, str)
434
435
        properties = self.get_properties(object_id)
436
        if self._data_store.dbus_interface == DS_DBUS_INTERFACE2:
437
            tree_id, parent_id = object_id
438
            res = self._data_store.save(tree_id, parent_id, properties, path,
439
                                        False, timeout=DBUS_TIMEOUT_MAX,
440
                                        byte_arrays=True)
441
            tree_id_, child_id = res
442
            assert tree_id == tree_id_
443
            return unicode(tree_id), unicode(child_id)
444
445
        else:
446
            self._data_store.update(object_id, properties, path, False,
447
                                    timeout=DBUS_TIMEOUT_MAX)
448
            return unicode(object_id)
449
450
    def _convert_metadata(self, metadata):
451
        """Convert metadata (as returned by the data store) to a unicode dict
452
453
        The data store may keep the data type it got as input or convert
454
        it to a string, at it's own discretion. To keep our processing
455
        sane and independent of the data store implementation, we pass
456
        unicode strings as input and convert output to unicode strings.
457
        As an exception, we keep values that cannot be converted from
458
        UTF-8 (e.g. previews in PNG format) as (binary) strings.
459
        """
460
        metadata_unicode = dict()
461
        for key, value in metadata.items():
462
            if isinstance(key, str):
463
                key_unicode = unicode(key, 'utf-8')
464
            else:
465
                key_unicode = unicode(key)
466
467
            if isinstance(value, str):
468
                try:
469
                    value_unicode = unicode(value, 'utf-8')
470
                except UnicodeDecodeError:
471
                    # Keep binary strings as-is
472
                    value_unicode = value
473
            else:
474
                value_unicode = unicode(value)
475
476
            metadata_unicode[key_unicode] = value_unicode
477
478
        return metadata_unicode
479
480
    def _change_metadata(self, object_id, metadata):
481
        if self._data_store.dbus_interface == DS_DBUS_INTERFACE2:
482
            tree_id, version_id = object_id
483
            self._data_store.change_metadata(tree_id, version_id, metadata)
484
485
        else:
486
            self._data_store.update(object_id, metadata, '', False)
487
488
489
class FSEntry(object):
490
    def __init__(self, file_system, mode):
491
        self._fs = file_system
492
        self._ds = file_system.data_store
493
        self.mode = mode
494
495
    def get_properties(self, names=None, use_cache=False):
496
        """Read the given properties (default: all)
497
498
        Returns a dictionary with unicode strings as keys and values.
499
        As an exception, values that cannot be converted from UTF-8
500
        (e.g. previews in PNG format) are represented by (binary)
501
        strings.
502
        """
503
        return []
504
505
    def list_properties(self):
506
        """List the names of all properties for this entry
507
508
        Returns a list of unicode strings.
509
        """
510
        return []
511
512
    def lookup(self, name_):
513
        raise IOError(errno.ENOENT, os.strerror(errno.ENOENT))
514
515
    def mkdir(self, name_):
516
        raise IOError(errno.EACCES, os.strerror(errno.EACCES))
517
518
    def readlink(self):
519
        raise IOError(errno.EINVAL, os.strerror(errno.EINVAL))
520
521
    def remove(self):
522
        """Remove this entry"""
523
        raise IOError(errno.EACCES, os.strerror(errno.EACCES))
524
525
    def create_property(self, name, value):
526
        """Set the given property, raising an error if it already exists"""
527
        raise IOError(errno.EOPNOTSUPP, os.strerror(errno.EOPNOTSUPP))
528
529
    def replace_property(self, name, value):
530
        """Modify the given, already existing property"""
531
        raise IOError(errno.EOPNOTSUPP, os.strerror(errno.EOPNOTSUPP))
532
533
    def set_properties(self, properties):
534
        """Write the given (sub)set of properties
535
536
        properties -- dictionary with unicode strings as keys and values
537
        """
538
        raise IOError(errno.EOPNOTSUPP, os.strerror(errno.EOPNOTSUPP))
539
540
    def remove_properties(self, names):
541
        """Remove the given (sub)set of properties
542
543
        names -- list of property names (unicode strings)
544
        """
545
        raise IOError(errno.EOPNOTSUPP, os.strerror(errno.EOPNOTSUPP))
546
547
    def get_ctime(self):
548
        """Return the time the object was originally created
549
550
        Return POSIX timestamp as float."""
551
        return 0.
552
553
    def get_mtime(self):
554
        """Return the time the object was last modified
555
556
        Return POSIX timestamp as float."""
557
        return time.time()
558
559
    def get_data(self):
560
        """Return the entire content of this entry"""
561
        # FIXME: inefficient or even impractical for large files
562
        raise IOError(errno.EISDIR, os.strerror(errno.EISDIR))
563
564
    def get_size(self):
565
        """Return the size of the content in bytes"""
566
        return 0
567
568
569
class Symlink(FSEntry):
570
    def __init__(self, file_system, target):
571
        assert isinstance(target, unicode)
572
        FSEntry.__init__(self, file_system, stat.S_IFLNK | 0777)
573
        self.target = target
574
575
    def readlink(self):
576
        return self.target
577
578
    def __repr__(self):
579
        return 'Symlink(%r, %r)' % (self._fs, self.target)
580
581
582
class DSObjectBase(FSEntry):
583
    def __init__(self, file_system, mode, object_id, metadata=None):
584
        FSEntry.__init__(self, file_system, mode)
585
        self.object_id = object_id
586
        self._metadata = metadata
587
        self._have_nonstandard = False
588
589
    def get_properties(self, names=None, use_cache=False):
590
        nonstandard_names = bool([True for name in names
591
                                  if name not in _USEFUL_PROPS])
592
        if names and not nonstandard_names:
593
            fetch_names = names
594
        else:
595
            fetch_names = None
596
        if ((not use_cache) or (self._metadata is None) or
597
            (nonstandard_names and not self._have_nonstandard)):
598
            self._metadata = self._ds.get_properties(self.object_id,
599
                                                     fetch_names)
600
            self._have_nonstandard = nonstandard_names
601
602
        if not names:
603
            return self._metadata
604
        return dict([(name, self._metadata[name])
605
                     for name in names
606
                     if name in self._metadata])
607
608
    def get_ctime(self):
609
        props = self.get_properties([u'creation_time', u'timestamp'])
610
        try:
611
            return float(props[u'creation_time'])
612
        except (KeyError, ValueError, TypeError):
613
            pass
614
615
        try:
616
            return float(props[u'timestamp'])
617
        except (KeyError, ValueError, TypeError):
618
            return time.time()
619
620
    def get_mtime(self):
621
        props = self.get_properties([u'creation_time', u'timestamp'])
622
        try:
623
            return float(props[u'timestamp'])
624
        except (KeyError, ValueError, TypeError):
625
            return time.time()
626
627
628
class ObjectSymlink(Symlink, DSObjectBase):
629
    def __init__(self, file_system, target, object_id, metadata=None):
630
        assert isinstance(target, unicode)
631
        DSObjectBase.__init__(self, file_system, stat.S_IFLNK | 0777,
632
                              object_id, metadata)
633
        self.target = target
634
635
636
class DSObject(DSObjectBase):
637
    def __init__(self, file_system, object_id, metadata=None):
638
        DSObjectBase.__init__(self, file_system, stat.S_IFREG | 0750,
639
                              object_id, metadata)
640
641
    def list_properties(self):
642
        return self._ds.list_properties(self.object_id)
643
644
    def create_property(self, name, value):
645
        return self._ds.create_property(self.object_id, name, value)
646
647
    def replace_property(self, name, value):
648
        return self._ds.replace_property(self.object_id, name, value)
649
650
    def set_properties(self, properties):
651
        return self._ds.set_properties(self.object_id, properties)
652
653
    def remove(self):
654
        self._ds.remove_entry(self.object_id)
655
656
    def remove_properties(self, names):
657
        return self._ds.remove_properties(self.object_id, names)
658
659
    def get_data(self):
660
        return self._ds.get_data(self.object_id)
661
662
    def write_data(self, file_name):
663
        return self._ds.write_data(self.object_id, file_name)
664
665
    def get_size(self):
666
        return self._ds.get_size(self.object_id)
667
668
669
class Directory(FSEntry):
670
    def __init__(self, file_system, level, mode, parent=None):
671
        self.parent = parent
672
        self._level = level
673
        FSEntry.__init__(self, file_system, stat.S_IFDIR | mode)
674
675
    def listdir(self):
676
        yield u'.'
677
        yield u'..'
678
679
    def readdir(self):
680
        yield (u'.', self)
681
        if self.parent is not None:
682
            yield (u'..', self.parent)
683
684
    def _get_symlink(self, object_id, metadata=None):
685
        directory_path = u'../' * self._level + u'by-id/'
686
        if isinstance(object_id, tuple):
687
            assert (isinstance(object_id[0], unicode) and
688
                    isinstance(object_id[1], unicode))
689
            return ObjectSymlink(self._fs,
690
                                 directory_path + u'/'.join(object_id),
691
                                 object_id, metadata)
692
        else:
693
            assert isinstance(object_id, unicode)
694
            return ObjectSymlink(self._fs, directory_path + object_id,
695
                                 object_id, metadata)
696
697
698
class ByTitleDirectory(Directory):
699
    def __init__(self, file_system, level, parent):
700
        Directory.__init__(self, file_system, level, 0550, parent)
701
702
    def listdir(self):
703
        for name in Directory.listdir(self):
704
            yield name
705
706
        for object_id in self._ds.list_object_ids():
707
            name = self._fs.lookup_title_name(object_id)
708
            yield name
709
710
    def readdir(self):
711
        for name, entry in Directory.readdir(self):
712
            yield name, entry
713
714
        for object_id, metadata in self._ds.list_metadata():
715
            name = self._fs.lookup_title_name(object_id, metadata)
716
            yield (name, self._get_symlink(object_id, metadata))
717
718
    def lookup(self, name):
719
        object_id = self._fs.resolve_title_name(name)
720
        return self._get_symlink(object_id)
721
722
    def mknod(self, name):
723
        if self._fs.try_resolve_title_name(name):
724
            raise IOError(errno.EEXIST, os.strerror(errno.EEXIST))
725
726
        object_id_ = self._ds.create_new({'title': name})
727
728
729
class ByUidDirectory(Directory):
730
    def __init__(self, file_system, level, parent):
731
        Directory.__init__(self, file_system, level, 0550, parent)
732
733
    def lookup(self, object_id):
734
        if not self._ds.check_object_id(object_id):
735
            raise IOError(errno.ENOENT, os.strerror(errno.ENOENT))
736
737
        return DSObject(self._fs, object_id)
738
739
    def listdir(self):
740
        for name in Directory.listdir(self):
741
            yield name
742
743
        for object_id in self._ds.list_object_ids():
744
            yield object_id
745
746
    def readdir(self):
747
        for name, entry in Directory.readdir(self):
748
            yield name, entry
749
750
        for object_id in self._ds.list_object_ids():
751
            yield (object_id, DSObject(self._fs, object_id))
752
753
754
class ByVersionIdDirectory(Directory):
755
    def __init__(self, file_system, level, parent, tree_id):
756
        self._tree_id = tree_id
757
        Directory.__init__(self, file_system, level, 0550, parent)
758
759
    def lookup(self, version_id):
760
        object_id = (self._tree_id, version_id)
761
        if not self._ds.check_object_id(object_id):
762
            raise IOError(errno.ENOENT, os.strerror(errno.ENOENT))
763
764
        return DSObject(self._fs, object_id)
765
766
    def listdir(self):
767
        for name in Directory.listdir(self):
768
            yield name
769
770
        for version_id in self._ds.list_versions(self._tree_id):
771
            yield (self._tree_id, version_id)
772
773
    def readdir(self):
774
        for name, entry in Directory.readdir(self):
775
            yield name, entry
776
777
        for version_id in self._ds.list_versions(self._tree_id):
778
            object_id = (self._tree_id, version_id)
779
            yield (object_id, DSObject(self._fs, object_id))
780
781
782
class ByTreeIdDirectory(Directory):
783
    def __init__(self, file_system, level, parent):
784
        Directory.__init__(self, file_system, level, 0550, parent)
785
786
    def lookup(self, tree_id):
787
        if not self._ds.check_tree_id(tree_id):
788
            raise IOError(errno.ENOENT, os.strerror(errno.ENOENT))
789
790
        return ByVersionIdDirectory(self._fs, self._level + 1, self, tree_id)
791
792
    def listdir(self):
793
        for name in Directory.listdir(self):
794
            yield name
795
796
        for tree_id in self._ds.list_tree_ids():
797
            yield tree_id
798
799
    def readdir(self):
800
        for name, entry in Directory.readdir(self):
801
            yield name, entry
802
803
        for tree_id in self._ds.list_tree_ids():
804
            yield (tree_id, ByVersionIdDirectory(self._fs, self._level + 1,
805
                                                 self, tree_id))
806
807
808
class ByTagsSubDirectory(ByTitleDirectory):
809
    def __init__(self, file_system, level, parent, tags):
810
        self._tags = frozenset(tags)
811
        ByTitleDirectory.__init__(self, file_system, level, parent)
812
813
    def mknod(self, name):
814
        if self._fs.try_resolve_title_name(name):
815
            raise IOError(errno.EEXIST, os.strerror(errno.EEXIST))
816
817
        props = {'title': name, 'tags': ' '.join(self._tags)}
818
        object_id_ = self._ds.create_new(props)
819
820
    def listdir(self):
821
        for name in Directory.listdir(self):
822
            yield name
823
824
        for object_id, metadata in self._find_entries():
825
            name = self._fs.lookup_title_name(object_id, metadata)
826
            yield name
827
828
    def readdir(self):
829
        for name, entry in Directory.readdir(self):
830
            yield name, entry
831
832
        for object_id, metadata in self._find_entries():
833
            name = self._fs.lookup_title_name(object_id, metadata)
834
            yield (name, self._get_symlink(object_id, metadata))
835
836
    def _find_entries(self):
837
        query = {'query': ' '.join(self._tags)}
838
        for object_id, props in self._ds.list_metadata(query):
839
            entry_tags = frozenset(props.get('tags', '').split())
840
            if self._tags - entry_tags:
841
                continue
842
843
            yield object_id, props
844
845
846
class ByTagsDirectory(Directory):
847
    def __init__(self, file_system, level, parent):
848
        Directory.__init__(self, file_system, level, 0550, parent)
849
        self._tag_dirs = {}
850
851
    def listdir(self):
852
        for name in Directory.listdir(self):
853
            yield name
854
855
        for tag in self._list_tags():
856
            if u'/' in tag or tag.startswith(u'.'):
857
                continue
858
859
            yield tag
860
861
    def readdir(self):
862
        for name, entry in Directory.readdir(self):
863
            yield name, entry
864
865
        for tag in self._list_tags():
866
            if u'/' in tag or tag.startswith(u'.'):
867
                continue
868
869
            if tag not in self._tag_dirs:
870
                self._tag_dirs[tag] = ByTagsSubDirectory(self._fs,
871
                                                         self._level + 1,
872
                                                         self, [tag])
873
            yield (tag, self._tag_dirs[tag])
874
875
    def lookup(self, name):
876
        if name not in self._tag_dirs:
877
            if not self._check_tag(name):
878
                raise IOError(errno.ENOENT, os.strerror(errno.ENOENT))
879
880
            self._tag_dirs[name] = ByTagsSubDirectory(self._fs,
881
                                                      self._level + 1, self,
882
                                                      [name])
883
        return self._tag_dirs[name]
884
885
    def _check_tag(self, name):
886
        return self._ds.check_property_contains(u'tags', name)
887
888
    def _list_tags(self):
889
        tags = set()
890
        for value in self._ds.list_property_values(u'tags'):
891
            tags.update((value or u'').split())
892
893
        tags.discard(u'')
894
        return tags
895
896
897
class RootDirectory(Directory):
898
    def __init__(self, file_system, mode):
899
        Directory.__init__(self, file_system, 0, mode, None)
900
        self._by_tags_directory = ByTagsDirectory(file_system, 1, self)
901
        self._by_title_directory = ByTitleDirectory(file_system, 1, self)
902
        if self._ds.supports_versions:
903
            self._by_id_directory = ByTreeIdDirectory(file_system, 1, self)
904
        else:
905
            self._by_id_directory = ByUidDirectory(file_system, 1, self)
906
907
    def listdir(self):
908
        for name in Directory.listdir(self):
909
            yield name
910
911
        yield u'by-id'
912
        yield u'by-tags'
913
        yield u'by-title'
914
915
    def readdir(self):
916
        for name, entry in Directory.readdir(self):
917
            yield name, entry
918
919
        yield (u'by-id', self._by_id_directory)
920
        yield (u'by-tags', self._by_tags_directory)
921
        yield (u'by-title', self._by_title_directory)
922
923
    def lookup(self, name):
924
        if name == u'by-id':
925
            return self._by_id_directory
926
        elif name == u'by-tags':
927
            return self._by_tags_directory
928
        elif name == u'by-title':
929
            return self._by_title_directory
930
931
        raise IOError(errno.ENOENT, os.strerror(errno.ENOENT))
932
933
934
class FSEmulation(object):
935
936
    # public API
937
938
    def __init__(self):
939
        self.data_store = DataStore()
940
        # FIXME: determine good LRU size
941
        self._cache = _LRU(500)
942
        self._root_dir = RootDirectory(self, 0550)
943
        self._object_id_to_title_name = {}
944
        self._title_name_to_object_id = {}
945
946
    def resolve(self, path, follow_links=False):
947
        assert isinstance(path, unicode)
948
949
        stripped_path = path.strip(u'/')
950
        if not stripped_path:
951
            return self._root_dir
952
953
        partial_path = u''
954
        entry = self._root_dir
955
        for component in stripped_path.split(u'/'):
956
            partial_path += u'/' + component
957
            # FIXME: add cache (in)validation
958
            if partial_path not in self._cache:
959
                self._cache[partial_path] = entry.lookup(component)
960
961
            entry = self._cache[partial_path]
962
963
        if path.endswith(u'/') and not isinstance(entry, Directory):
964
            raise IOError(errno.ENOTDIR, os.strerror(errno.ENOTDIR))
965
966
        if isinstance(entry, Symlink) and follow_links:
967
            target = u'/%s/../%s' % (stripped_path, entry.readlink())
968
            target_abs = os.path.abspath(target)
969
            return self.resolve(target_abs, follow_links=True)
970
971
        return entry
972
973
    # internal API (for FSEntry derivatives)
974
975
    def resolve_title_name(self, name):
976
        if name not in self._title_name_to_object_id:
977
            # FIXME: Hack to fill self._title_name_to_object_id. To be
978
            # replaced by parsing the name and doing a specific search.
979
            list(self.resolve(u'/by-title').readdir())
980
981
        try:
982
            return self._title_name_to_object_id[name]
983
984
        except KeyError:
985
            raise IOError(errno.ENOENT, os.strerror(errno.ENOENT))
986
987
    def try_resolve_title_name(self, name):
988
        return self._title_name_to_object_id.get(name)
989
990
    def lookup_title_name(self, object_id, metadata=None):
991
        name = self._object_id_to_title_name.get(object_id)
992
        if name:
993
            return name
994
995
        if metadata is None:
996
            metadata = self.data_store.get_properties(object_id,
997
                                                      _USEFUL_PROPS)
998
999
        name = self._generate_title_name(metadata, object_id)
1000
        self._add_title_name(name, object_id)
1001
        return name
1002
1003
    # private methods
1004
1005
    def _add_title_name(self, name, object_id):
1006
        self._object_id_to_title_name[object_id] = name
1007
        self._title_name_to_object_id[name] = object_id
1008
        return name
1009
1010
    def _generate_title_name(self, metadata, object_id):
1011
        title = metadata.get(u'title')
1012
        try:
1013
            mtime = float(metadata[u'timestamp'])
1014
        except (KeyError, ValueError):
1015
            mtime = time.time()
1016
1017
        time_human = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(mtime))
1018
        name = u'%s - %s' % (title, time_human)
1019
        name = safe_name(name)
1020
        extension = self._guess_extension(metadata.get('mime_type'), object_id)
1021
        if extension:
1022
            current_name = u'%s.%s' % (name, extension)
1023
        else:
1024
            current_name = name
1025
        counter = 1
1026
        while current_name in self._title_name_to_object_id:
1027
            counter += 1
1028
            if extension:
1029
                current_name = u'%s %d.%s' % (name, counter, extension)
1030
            else:
1031
                current_name = u'%s %d' % (name, counter)
1032
1033
        return current_name
1034
1035
    def _remove_title_name_by_object_id(self, object_id):
1036
        name = self._object_id_to_title_name.pop(object_id, None)
1037
        if name:
1038
            del self._title_name_to_object_id[name]
1039
1040
    def _remove_title_name_by_name(self, name):
1041
        object_id = self._title_name_to_object_id.pop(name, None)
1042
        if object_id:
1043
            del self._object_id_to_title_name[object_id]
1044
1045
    def _guess_extension(self, mime_type, object_id):
1046
        extension = None
1047
1048
        if not mime_type:
1049
            file_name = self.data_store.get_data(object_id)
1050
            if file_name:
1051
                try:
1052
                    mime_type = sugar.mime.get_for_file(file_name)
1053
                finally:
1054
                    os.remove(file_name)
1055
1056
        if mime_type:
1057
            extension = sugar.mime.get_primary_extension(mime_type)
1058
1059
        return extension
1060
1061
1062
def safe_name(name):
1063
    return name.replace(u'/', u'_')